From a66affa788bb13da2e6698b964a332c2b91e1e8c Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Wed, 22 Jun 2022 16:42:21 +0200 Subject: Merging similar 2D aoi into a single 2D scene. --- src/argaze/AreaOfInterest/AOI2DScene.py | 4 +-- .../export_tobii_segment_aruco_visual_scan.py | 23 ++++++++++--- .../utils/live_tobii_aruco_aoi_ivy_controller.py | 39 +++++++++++++++------- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/argaze/AreaOfInterest/AOI2DScene.py b/src/argaze/AreaOfInterest/AOI2DScene.py index b009f64..1951cf4 100644 --- a/src/argaze/AreaOfInterest/AOI2DScene.py +++ b/src/argaze/AreaOfInterest/AOI2DScene.py @@ -35,7 +35,7 @@ class AOI2DScene(AOIFeatures.AOIScene): return looked, ignored - def draw(self, frame, gaze_position: GazeFeatures.GazePosition, exclude=[]): + def draw(self, frame, gaze_position: GazeFeatures.GazePosition, exclude=[], base_color=(0, 0, 255), looked_color=(0, 255, 0)): """Draw AOI polygons on frame.""" for name, aoi2D in self.items(): @@ -45,7 +45,7 @@ class AOI2DScene(AOIFeatures.AOIScene): looked = aoi2D.looked(gaze_position) - color = (0, 255, 0) if looked else (0, 0, 255) + color = looked_color if looked else base_color if looked: top_left_corner_pixel = numpy.rint(aoi2D.clockwise()[0]).astype(int) diff --git a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py index 1978b44..04e21d6 100644 --- a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py +++ b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py @@ -208,6 +208,9 @@ def main(): # Project 3D scene on each video frame and the visualisation frame if aruco_tracker.get_markers_number(): + # Store aoi 2D video for further scene merging + aoi2D_dict = {} + for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()): # Select 3D scene related to detected marker @@ -223,11 +226,13 @@ def main(): # This hack isn't realistic but as the gaze will mainly focus on centered AOI, where the distorsion is low, it is acceptable. aoi2D_video_scene = aoi3D_scene.project(aruco_camera.get_K()) - # Draw 2D scene on video frame excluding 'Visualisation_Plan' aoi - aoi2D_video_scene.draw(video_frame.matrix, video_gaze_pixel, ['Visualisation_Plan']) + # Store each 2D aoi for further scene merging + for name, aoi in aoi2D_video_scene.items(): + + if name not in aoi2D_dict.keys(): + aoi2D_dict[name] = [] - # Store 2D scene at this time in millisecond - ts_aois_scenes[round(video_ts_ms)] = aoi2D_video_scene + aoi2D_dict[name].append(aoi.clockwise()) # Select 2D visu scene if there is one for the detected marker aoi2D_visu_scene = aoi2D_visu_scene_selector(marker_id) @@ -240,7 +245,17 @@ def main(): visu_gaze_pixel = aoi2D_visu_scene['Visualisation_Plan'].looked_pixel(look_at) cv.circle(aoi2D_visu_frame, visu_gaze_pixel, 4, (0, 0, 255), -1) + + # Merge all 2D aoi into a single 2D scene + aoi2D_merged_scene = AOI2DScene.AOI2DScene() + for name, aoi_array in aoi2D_dict.items(): + aoi2D_merged_scene[name] = numpy.sum(aoi_array, axis=0) / len(aoi_array) + + aoi2D_merged_scene.draw(video_frame.matrix, video_gaze_pixel, exclude=['Visualisation_Plan']) + # Store 2D merged scene at this time in millisecond + ts_aois_scenes[round(video_ts_ms)] = aoi2D_merged_scene + if args.window: # Close window using 'Esc' key diff --git a/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py b/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py index 2218bb5..757ba9c 100644 --- a/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py +++ b/src/argaze/utils/live_tobii_aruco_aoi_ivy_controller.py @@ -47,7 +47,7 @@ def main(): tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'myProject', 'mySelf') # Calibrate tobii glasses - #tobii_controller.calibrate() + tobii_controller.calibrate() # Enable tobii data stream tobii_data_stream = tobii_controller.enable_data_stream() @@ -117,6 +117,9 @@ def main(): # Project 3D scenes related to each aruco markers if aruco_tracker.get_markers_number(): + # Store aoi 2D video for further scene merging + aoi2D_dict = {} + for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()): # Select 3D scene related to detected marker @@ -133,20 +136,32 @@ def main(): # DON'T APPLY CAMERA DISTORSION : it projects points which are far from the frame into it # This hack isn't realistic but as the gaze will mainly focus on centered AOI, where the distorsion is low, it is acceptable. - aoi2D_scene = aoi3D_scene.project(aruco_camera.get_K(), D0) + aoi2D_video_scene = aoi3D_scene.project(aruco_camera.get_K(), D0) + + # Store each 2D aoi for further scene merging + for name, aoi in aoi2D_video_scene.items(): + + if name not in aoi2D_dict.keys(): + aoi2D_dict[name] = [] + + aoi2D_dict[name].append(aoi.clockwise()) + + # Merge all 2D aoi into a single 2D scene + aoi2D_merged_scene = AOI2DScene.AOI2DScene() + for name, aoi_array in aoi2D_dict.items(): + aoi2D_merged_scene[name] = numpy.sum(aoi_array, axis=0) / len(aoi_array) - # Draw 2D scene - aoi2D_scene.draw(video_frame.matrix, video_gaze_pixel) + aoi2D_merged_scene.draw(video_frame.matrix, video_gaze_pixel, exclude=['Visualisation_Plan']) - # Send look at aoi pointer - for name, aoi in aoi2D_scene.items(): - if aoi.looked(video_gaze_pixel): + # Send look at aoi pointer + for name, aoi in aoi2D_merged_scene.items(): + if aoi.looked(video_gaze_pixel): - # 4 corners aoi - if len(aoi) == 4: - IvySendMsg(f'looking {name} at {aoi.look_at(video_gaze_pixel)}') - else: - IvySendMsg(f'looking {name}') + # 4 corners aoi + if len(aoi) == 4: + IvySendMsg(f'looking {name} at {aoi.look_at(video_gaze_pixel)}') + else: + IvySendMsg(f'looking {name}') # Close window using 'Esc' key if cv.waitKey(1) == 27: -- cgit v1.1