Merging similar 2D aoi into a single 2D scene.

3 files changed, 48 insertions, 18 deletions

diff --git a/src/argaze/AreaOfInterest/AOI2DScene.py b/src/argaze/AreaOfInterest/AOI2DScene.py
index b009f64..1951cf4 100644
--- a/src/argaze/AreaOfInterest/AOI2DScene.py
+++ b/src/argaze/AreaOfInterest/AOI2DScene.py
@@ -35,7 +35,7 @@ class AOI2DScene(AOIFeatures.AOIScene):
 
 		return looked, ignored
 
-	def draw(self, frame, gaze_position: GazeFeatures.GazePosition, exclude=[]):
+	def draw(self, frame, gaze_position: GazeFeatures.GazePosition, exclude=[], base_color=(0, 0, 255), looked_color=(0, 255, 0)):
 		"""Draw AOI polygons on frame."""
 
 		for name, aoi2D in self.items():
@@ -45,7 +45,7 @@ class AOI2DScene(AOIFeatures.AOIScene):
 
 			looked = aoi2D.looked(gaze_position)
 
-			color = (0, 255, 0) if looked else (0, 0, 255)
+			color = looked_color if looked else base_color
 
 			if looked:
 				top_left_corner_pixel = numpy.rint(aoi2D.clockwise()[0]).astype(int)
diff --git a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py
index 1978b44..04e21d6 100644
--- a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py
+++ b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py
@@ -208,6 +208,9 @@ def main():
                 # Project 3D scene on each video frame and the visualisation frame
                 if aruco_tracker.get_markers_number():
 
+                    # Store aoi 2D video for further scene merging
+                    aoi2D_dict = {}
+
                     for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()):
 
                         # Select 3D scene related to detected marker
@@ -223,11 +226,13 @@ def main():
                         # This hack isn't realistic but as the gaze will mainly focus on centered AOI, where the distorsion is low, it is acceptable.
                         aoi2D_video_scene = aoi3D_scene.project(aruco_camera.get_K())
 
-                        # Draw 2D scene on video frame excluding 'Visualisation_Plan' aoi
-                        aoi2D_video_scene.draw(video_frame.matrix, video_gaze_pixel, ['Visualisation_Plan'])
+                        # Store each 2D aoi for further scene merging
+                        for name, aoi in aoi2D_video_scene.items():
+
+                            if name not in aoi2D_dict.keys():
+                                aoi2D_dict[name] = []
 
-                        # Store 2D scene at this time in millisecond
-                        ts_aois_scenes[round(video_ts_ms)] = aoi2D_video_scene
+                            aoi2D_dict[name].append(aoi.clockwise())
 
                         # Select 2D visu scene if there is one for the detected marker
                         aoi2D_visu_scene = aoi2D_visu_scene_selector(marker_id)
@@ -240,7 +245,17 @@ def main():
 
                         visu_gaze_pixel = aoi2D_visu_scene['Visualisation_Plan'].looked_pixel(look_at)
                         cv.circle(aoi2D_visu_frame, visu_gaze_pixel, 4, (0, 0, 255), -1)
+
+                    # Merge all 2D aoi into a single 2D scene
+                    aoi2D_merged_scene = AOI2DScene.AOI2DScene()
+                    for name, aoi_array in aoi2D_dict.items():
+                        aoi2D_merged_scene[name] = numpy.sum(aoi_array, axis=0) / len(aoi_array)
+
+                    aoi2D_merged_scene.draw(video_frame.matrix, video_gaze_pixel, exclude=['Visualisation_Plan'])
                         
+                    # Store 2D merged scene at this time in millisecond
+                    ts_aois_scenes[round(video_ts_ms)] = aoi2D_merged_scene
+
                 if args.window:
 
                     # Close window using 'Esc' key
diff --git a/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py b/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py
index 2218bb5..757ba9c 100644
--- a/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py
+++ b/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py
@@ -47,7 +47,7 @@ def main():
     tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'myProject', 'mySelf')
 
     # Calibrate tobii glasses
-    #tobii_controller.calibrate()
+    tobii_controller.calibrate()
 
     # Enable tobii data stream
     tobii_data_stream = tobii_controller.enable_data_stream()
@@ -117,6 +117,9 @@ def main():
             # Project 3D scenes related to each aruco markers
             if aruco_tracker.get_markers_number():
 
+                # Store aoi 2D video for further scene merging
+                aoi2D_dict = {}
+
                 for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()):
 
                     # Select 3D scene related to detected marker
@@ -133,20 +136,32 @@ def main():
 
                     # DON'T APPLY CAMERA DISTORSION : it projects points which are far from the frame into it
                     # This hack isn't realistic but as the gaze will mainly focus on centered AOI, where the distorsion is low, it is acceptable.
-                    aoi2D_scene = aoi3D_scene.project(aruco_camera.get_K(), D0)
+                    aoi2D_video_scene = aoi3D_scene.project(aruco_camera.get_K(), D0)
+
+                    # Store each 2D aoi for further scene merging
+                    for name, aoi in aoi2D_video_scene.items():
+
+                        if name not in aoi2D_dict.keys():
+                            aoi2D_dict[name] = []
+
+                        aoi2D_dict[name].append(aoi.clockwise())
+
+                # Merge all 2D aoi into a single 2D scene
+                aoi2D_merged_scene = AOI2DScene.AOI2DScene()
+                for name, aoi_array in aoi2D_dict.items():
+                    aoi2D_merged_scene[name] = numpy.sum(aoi_array, axis=0) / len(aoi_array)
 
-                    # Draw 2D scene
-                    aoi2D_scene.draw(video_frame.matrix, video_gaze_pixel)
+                aoi2D_merged_scene.draw(video_frame.matrix, video_gaze_pixel, exclude=['Visualisation_Plan'])
 
-                    # Send look at aoi pointer
-                    for name, aoi in aoi2D_scene.items():
-                        if aoi.looked(video_gaze_pixel):
+                # Send look at aoi pointer
+                for name, aoi in aoi2D_merged_scene.items():
+                    if aoi.looked(video_gaze_pixel):
 
-                            # 4 corners aoi
-                            if len(aoi) == 4:
-                                IvySendMsg(f'looking {name} at {aoi.look_at(video_gaze_pixel)}')
-                            else:
-                                IvySendMsg(f'looking {name}')
+                        # 4 corners aoi
+                        if len(aoi) == 4:
+                            IvySendMsg(f'looking {name} at {aoi.look_at(video_gaze_pixel)}')
+                        else:
+                            IvySendMsg(f'looking {name}')
 
             # Close window using 'Esc' key
             if cv.waitKey(1) == 27: