Updating empty scene handling. Removing visual scan processing.

1 files changed, 26 insertions, 130 deletions

diff --git a/src/argaze/utils/tobii_segment_aruco_aoi_export.py b/src/argaze/utils/tobii_segment_aruco_aoi_export.py
index a3a31d0..b1fe4ed 100644
--- a/src/argaze/utils/tobii_segment_aruco_aoi_export.py
+++ b/src/argaze/utils/tobii_segment_aruco_aoi_export.py
@@ -18,8 +18,7 @@ def main():
     """
     Track ArUco markers into Tobii Glasses Pro 2 segment video file. 
     For each loaded AOI scene .obj file, position the scene virtually relatively to each detected ArUco markers and project the scene into camera frame. 
-    Then, detect if Tobii gaze point is inside any AOI.
-    Export AOIs video and data.
+    Export AOIs video and data as a aruco_aoi.csv, aruco_aoi.mp4 files
     """
 
     # Manage arguments
@@ -71,9 +70,8 @@ def main():
                     os.makedirs(destination_path)
                     print(f'{destination_path} folder created')
 
-        vs_data_filepath = f'{destination_path}/visual_scan.csv'
-        vs_visu_filepath = f'{destination_path}/visual_scan_marker_%d.jpg'
-        vs_video_filepath = f'{destination_path}/visual_scan.mp4'
+        vs_data_filepath = f'{destination_path}/aruco_aoi.csv'
+        vs_video_filepath = f'{destination_path}/aruco_aoi.mp4'
 
         # Load a tobii segment
         tobii_segment = TobiiEntities.TobiiSegment(args.segment_path, int(args.time_range[0] * 1e6), int(args.time_range[1] * 1e6) if args.time_range[1] != None else None)
@@ -89,12 +87,6 @@ def main():
         for name in tobii_segment_data.keys():
             print(f'\t{name}: {len(tobii_segment_data[name])} data')
 
-        # Access to timestamped gaze positions data buffer
-        tobii_ts_gaze_positions = tobii_segment_data['GazePosition']
-
-        # Access to timestamped gaze 3D positions data buffer
-        tobii_ts_gaze_positions_3d = tobii_segment_data['GazePosition3D']
-
         # Access to timestamped head rotations data buffer
         tobii_ts_head_rotations = tobii_segment_data['Gyroscope']
 
@@ -111,7 +103,7 @@ def main():
 
         else:
 
-            raise ValueError('.json camera calibration filepath required. Use -c option.')
+            raise UserWarning('.json camera calibration filepath required. Use -c option.')
 
         # Create aruco tracker
         aruco_tracker = ArUcoTracker.ArUcoTracker(args.marker_dictionary, args.marker_size, aruco_camera)
@@ -139,53 +131,14 @@ def main():
             print(f'AOI in {os.path.basename(aoi_scene_filepath)} scene related to marker #{marker_id}:')
             for aoi in aoi3D_scenes[marker_id].keys():
 
-                # If a 'Visualisation_Plan' AOI exist
-                # TODO: document this deep feature !!!
-                if aoi == 'Visualisation_Plan':
-
-                    print(f'\tVisualisation_Plan detected: a visual scan picture will be output for this marker.')
-
-                    # Create a visual scan visualisation frame
-                    visu_width, visu_height = 1920, 1080
-                    scene_width, scene_height, __ = aoi3D_scenes[marker_id].size()
-
-                    aoi2D_visu_frames[marker_id] = numpy.full((visu_height, visu_width, 3), 255, dtype=numpy.uint8)
-
-                    if args.time_range != (0., None):
-                        cv.putText(aoi2D_visu_frames[marker_id], f'Segment time range: {int(args.time_range[0] * 1000)} - {int(args.time_range[1] * 1000)} ms', (20, 40), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 1, cv.LINE_AA)
-
-                    # Project 3D scene onto the visualisation plan
-                    aoi3D_scene_rotation = numpy.array([[-numpy.pi, 0.0, 0.0]])
-                    aoi3D_scene_translation = aoi3D_scenes[marker_id].center()*[-1, 1, 0] + [0, 0, scene_height]
-
-                    # Edit a projection matrix for the reference frame
-                    K0 = numpy.array([[visu_height, 0.0, visu_width/2], [0.0, visu_height, visu_height/2], [0.0, 0.0, 1.0]])
-
-                    aoi2D_visu_scenes[marker_id] = aoi3D_scenes[marker_id].project(aoi3D_scene_translation, aoi3D_scene_rotation, K0)
-
-                    for name, aoi in aoi2D_visu_scenes[marker_id].items():
-                        if name != 'Visualisation_Plan':
-                            aoi.draw(aoi2D_visu_frames[marker_id], (0, 0, 0))
-
-                else:
-
-                    print(f'\t{aoi}')
+                print(f'\t{aoi}')
 
         def aoi3D_scene_selector(marker_id):
             return aoi3D_scenes.get(marker_id, None)
 
-        def aoi2D_visu_scene_selector(marker_id):
-            return aoi2D_visu_scenes.get(marker_id, None)
-
-        def aoi2D_visu_frame_selector(marker_id):
-            return aoi2D_visu_frames.get(marker_id, None)
-
         # Create timestamped buffer to store AOIs scene in time
         ts_aois_scenes = AOIFeatures.TimeStampedAOIScenes()
 
-        # Create timestamped buffer to store gaze positions in time
-        ts_gaze_positions = GazeFeatures.TimeStampedGazePositions()
-
         # !!! the parameters below are specific to the TobiiGlassesPro2 !!!
         # Reference : https://www.biorxiv.org/content/10.1101/299925v1
         tobii_accuracy = 1.42 # degree
@@ -234,37 +187,10 @@ def main():
 
                     # When head is moving, ArUco tracking could return bad pose estimation and so bad AOI scene projection
                     if head_moving:
-                        raise AOIFeatures.AOISceneMissing('Head is moving')
-
-                    # Get nearest gaze position before video timestamp and remove all gaze positions before
-                    _, nearest_gaze_position = tobii_ts_gaze_positions.pop_first_until(video_ts)
-
-                    # Ignore frame when gaze position is not valid
-                    if nearest_gaze_position.validity == 1:
-                        raise GazeFeatures.GazePositionMissing('Unvalid gaze position')
-
-                    gaze_position_pixel = GazeFeatures.GazePosition( (int(nearest_gaze_position.value[0] * visu_frame.width), int(nearest_gaze_position.value[1] * visu_frame.height)) )
-
-                    # Draw gaze position
-                    cv.circle(visu_frame.matrix, gaze_position_pixel, 2, (0, 255, 255), -1)
-
-                    # Get nearest gaze position 3D before video timestamp and remove all gaze positions before
-                    _, nearest_gaze_position_3d = tobii_ts_gaze_positions_3d.pop_first_until(video_ts)
 
-                    # Ignore frame when gaze position 3D is not valid
-                    if nearest_gaze_position_3d.validity == 1:
-                        raise GazeFeatures.GazePositionMissing('Unvalid gaze position 3D')
-
-                    gaze_accuracy_mm = numpy.tan(numpy.deg2rad(tobii_accuracy)) * nearest_gaze_position_3d.value[2]
-                    tobii_camera_hfov_mm = numpy.tan(numpy.deg2rad(tobii_camera_hfov / 2)) * nearest_gaze_position_3d.value[2]
-
-                    gaze_position_pixel.accuracy = round(visu_frame.width * float(gaze_accuracy_mm) / float(tobii_camera_hfov_mm))
-
-                    # Draw gaze accuracy
-                    cv.circle(visu_frame.matrix, gaze_position_pixel, gaze_position_pixel.accuracy, (0, 255, 255), 1)
-
-                    # Store gaze position in millisecond for further visual scan processing
-                    ts_gaze_positions[round(video_ts_ms)] = gaze_position_pixel
+                        ts_aois_scenes[round(video_ts_ms)] = AOIFeatures.EmptyAOIScene()
+                        
+                        raise UserWarning('Head is moving')
 
                     # Hide frame left and right borders before tracking to ignore markers outside focus area
                     cv.rectangle(video_frame.matrix, (0, 0), (int(video_frame.width/6), int(video_frame.height)), (0, 0, 0), -1)
@@ -276,7 +202,10 @@ def main():
 
                     # When no marker is detected, no AOI scene projection can't be done
                     if aruco_tracker.get_markers_number() == 0:
-                        raise AOIFeatures.AOISceneMissing('No marker detected')
+
+                        ts_aois_scenes[round(video_ts_ms)] = AOIFeatures.EmptyAOIScene()
+
+                        raise UserWarning('No marker detected')
 
                     # Store aoi 2D video for further scene merging
                     aoi2D_dict = {}
@@ -294,7 +223,7 @@ def main():
                         aoi3D_camera = aoi3D_scene.transform(aruco_tracker.get_marker_translation(i), aruco_tracker.get_marker_rotation(i))
 
                         # Get aoi inside vision cone field 
-                        cone_vision_height_cm = nearest_gaze_position_3d.value[2]/10 # cm
+                        cone_vision_height_cm = 200 # cm
                         cone_vision_radius_cm = numpy.tan(numpy.deg2rad(tobii_visual_hfov / 2)) * cone_vision_height_cm
 
                         aoi3D_inside, aoi3D_outside = aoi3D_camera.vision_cone(cone_vision_radius_cm, cone_vision_height_cm)
@@ -314,51 +243,28 @@ def main():
 
                             aoi2D_dict[name].append(aoi.clockwise())
 
-                        # Select 2D visu scene if there is one for the detected marker
-                        aoi2D_visu_scene = aoi2D_visu_scene_selector(marker_id)
-                        aoi2D_visu_frame = aoi2D_visu_frame_selector(marker_id)
-                        
-                        if aoi2D_visu_scene == None:
-                            continue
-                        
-                        look_at = aoi2D_video_scene['Visualisation_Plan'].look_at(gaze_position_pixel)
-
-                        visu_gaze_pixel = aoi2D_visu_scene['Visualisation_Plan'].looked_pixel(look_at)
-                        cv.circle(aoi2D_visu_frame, visu_gaze_pixel, 4, (0, 0, 255), -1)
-
                     # Merge all 2D aoi into a single 2D scene
                     aoi2D_merged_scene = AOI2DScene.AOI2DScene()
                     for name, aoi_array in aoi2D_dict.items():
                         aoi2D_merged_scene[name] = numpy.sum(aoi_array, axis=0) / len(aoi_array)
 
-                    aoi2D_merged_scene.draw(visu_frame.matrix, gaze_position_pixel, exclude=['Visualisation_Plan'])
+                    aoi2D_merged_scene.draw(visu_frame.matrix, (0, 0))
 
-                    # When the merged scene is empty
-                    if len(aoi2D_merged_scene.keys()) == 0:
-                        raise AOIFeatures.AOISceneMissing('Scene is empty')
-                        
                     # Store 2D merged scene at this time in millisecond
                     ts_aois_scenes[round(video_ts_ms)] = aoi2D_merged_scene
 
-                # Raised when gaze data is missing
-                except GazeFeatures.GazePositionMissing as e:
-
-                    # Store missing gaze data exception
-                    ts_gaze_positions[round(video_ts_ms)] = e
-                    
-                    cv.rectangle(visu_frame.matrix, (0, 50), (550, 100), (127, 127, 127), -1)
-                    cv.putText(visu_frame.matrix, str(e), (20, 80), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
+                    # Warn user when the merged scene is empty
+                    if len(aoi2D_merged_scene.keys()) == 0:
 
-                # Raised when aoi scene is missing
-                except AOIFeatures.AOISceneMissing as e:
+                        raise UserWarning('Scene is empty')
 
-                    # Store missing scene exception
-                    ts_aois_scenes[round(video_ts_ms)] = e
+                # Write warning
+                except UserWarning as w:
 
                     cv.rectangle(visu_frame.matrix, (0, 50), (550, 100), (127, 127, 127), -1)
-                    cv.putText(visu_frame.matrix, str(e), (20, 80), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
+                    cv.putText(visu_frame.matrix, str(w), (20, 80), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
 
-                # Raised when buffer is empty
+                # Raised when timestamped buffer is empty
                 except ValueError:
                     pass
 
@@ -413,22 +319,12 @@ def main():
         for marker_id, rejected_count in rejected_counts.items():
             print(f'Markers {marker_id} has been rejected in {rejected_count} / {try_count} frames ({round(100 * rejected_count / try_count, 2)} %)')
 
-        # Build visual scan based on a pointer position
-        visual_scan = GazeFeatures.PointerBasedVisualScan(ts_aois_scenes, ts_gaze_positions)
-        print(f'{len(visual_scan.steps())} visual scan steps found')
-
-        # Export visual scan data
-        visual_scan.export_as_csv(vs_data_filepath)
-        print(f'Visual scan data saved into {vs_data_filepath}')
-
-        # Export each visual scan picture
-        for marker_id, aoi2D_visu_frame in aoi2D_visu_frames.items():
-            cv.imwrite(vs_visu_filepath % marker_id, visu_frame.matrix)
-            print(f'Visual scan picture for marker {marker_id} saved into {vs_visu_filepath % marker_id}')
-
-        # Notify when the visual scan video has been exported
-        print(f'Visual scan video saved into {vs_video_filepath}')
+        # Export aruco aoi data
+        ts_aois_scenes.export_as_csv(vs_data_filepath, exclude=['dimension'])
+        print(f'Aruco AOI data saved into {vs_data_filepath}')
 
+        # Notify when the aruco aoi video has been exported
+        print(f'Aruco AOI video saved into {vs_video_filepath}')
 
 if __name__ == '__main__':