Displaying unknown movements.

1 files changed, 222 insertions, 106 deletions

diff --git a/src/argaze/utils/tobii_segment_gaze_movements_export.py b/src/argaze/utils/tobii_segment_gaze_movements_export.py
index 88f61d7..7e184ee 100644
--- a/src/argaze/utils/tobii_segment_gaze_movements_export.py
+++ b/src/argaze/utils/tobii_segment_gaze_movements_export.py
@@ -62,6 +62,7 @@ def main():
 
     fixations_filepath = f'{destination_path}/gaze_fixations.csv'
     saccades_filepath = f'{destination_path}/gaze_saccades.csv'
+    unknown_filepath = f'{destination_path}/gaze_unknown.csv'
 
     gaze_status_filepath = f'{destination_path}/gaze_status.csv'
     gaze_status_video_filepath = f'{destination_path}/gaze_status.mp4'
@@ -109,16 +110,17 @@ def main():
     ts_projection_metrics = DataStructures.TimeStampedBuffer()
 
     # Starting with no AOI projection
-    selected_aoi = AOIFeatures.AreaOfInterest()
+    ts_current_aoi = 0
+    current_aoi = AOIFeatures.AreaOfInterest()
 
     # Initialise progress bar
-    MiscFeatures.printProgressBar(0, tobii_segment_video.duration, prefix = '\nGazePositions projection:', suffix = 'Complete', length = 100)
+    #MiscFeatures.printProgressBar(0, tobii_segment_video.duration, prefix = '\nGazePositions projection:', suffix = 'Complete', length = 100)
 
     for ts, tobii_gaze_position in tobii_ts_gaze_positions.items():
 
         # Update Progress Bar
         progress = ts - int(args.time_range[0] * 1e6)
-        MiscFeatures.printProgressBar(progress, tobii_segment_video.duration, prefix = 'GazePositions projection:', suffix = 'Complete', length = 100)
+        #MiscFeatures.printProgressBar(progress, tobii_segment_video.duration, prefix = 'GazePositions projection:', suffix = 'Complete', length = 100)
 
         try:
 
@@ -127,34 +129,42 @@ def main():
 
             assert(ts_current_aois <= ts)
 
-            # Is the AOI projection valid ?
+            # Catch aoi error to not update current aoi
             if 'error' in current_aois.keys():
 
                 # TODO: display error 
                 current_aoi_error = current_aois.pop('error')
 
-                # Wait for valid aoi projection
-                continue
+            # Or update current aoi
+            elif args.aoi in current_aois.keys():
 
-            # Is the selected aoi there ?
-            if args.aoi in current_aois.keys():
+                ts_current_aoi = ts_current_aois
+                current_aoi = AOIFeatures.AreaOfInterest(current_aois.pop(args.aoi))
 
-                selected_aoi = AOIFeatures.AreaOfInterest(current_aois.pop(args.aoi))
-                selected_ts_aoi = ts_current_aois
+        # No aoi projection at the beginning
+        except KeyError as e:
+            pass
 
-            # else use last one if exist
-            elif selected_aoi.empty:
-                continue
+        # Wait for available aoi
+        if current_aoi.empty:
+            
+            ts_gaze_positions[ts] = GazeFeatures.UnvalidGazePosition()
+            continue
+            
+        # QUESTION: What todo if the current aoi is too old ?
+        # if the aoi didn't move it is not a problem...
+        # For the moment, we avoid 1s old aoi and we provide a metric to assess the problem
+        ts_difference = ts - ts_current_aoi
 
-            # QUESTION: What todo if the current valid aoi is too old ?
-            # if the aoi didn't move it is not a problem...
-            # For the moment, we just provide a metric to assess its not too big
-            ts_projection_metrics[ts] = {'frame': selected_ts_aoi, 'age': ts - selected_ts_aoi}
+        # If aoi is not updated after the 
+        if ts_difference >= args.duration_threshold*1e3:
 
-        # Wait for a first aoi projection
-        except KeyError as e:
+            current_aoi = AOIFeatures.AreaOfInterest()
+            ts_gaze_positions[ts] = GazeFeatures.UnvalidGazePosition()
             continue
 
+        ts_projection_metrics[ts] = {'frame': ts_current_aois, 'age': ts_difference}
+
         # Test gaze position validity
         if tobii_gaze_position.validity == 0:
 
@@ -175,20 +185,18 @@ def main():
                 gaze_position = GazeFeatures.GazePosition(gaze_position_px, precision=gaze_precision_px)
 
                 # Project gaze position into selected aois
-                if selected_aoi.contains_point(gaze_position.value):
+                if current_aoi.contains_point(gaze_position.value):
 
-                    inner_x, inner_y = selected_aoi.inner_axis(gaze_position.value)
-                    inner_precision_px = gaze_precision_px * tobii_segment_video.width * tobii_segment_video.height / selected_aoi.area
+                    inner_x, inner_y = current_aoi.inner_axis(gaze_position.value)
+                    inner_precision_px = gaze_precision_px * tobii_segment_video.width * tobii_segment_video.height / current_aoi.area
 
                     # Store inner gaze position for further movement processing
                     # TEMP: 1920x1080 are Screen_Plan dimensions
-                    # TODO? trunc ts at the millisecond before int(ts*1e-3)*1e3
                     ts_gaze_positions[ts] = GazeFeatures.GazePosition((round(inner_x*1920), round(inner_y*1080)))#, precision=inner_precision_px) 
 
         # Store unvalid gaze position for further movement processing
         else:
 
-            # TODO? trunc ts at the millisecond before int(ts*1e-3)*1e3
             ts_gaze_positions[ts] = GazeFeatures.UnvalidGazePosition()
 
     print(f'\nGazePositions projection metrics:')
@@ -204,10 +212,11 @@ def main():
     movement_identifier = DispersionBasedGazeMovementIdentifier.GazeMovementIdentifier(args.dispersion_threshold, args.duration_threshold*1e3)
     ts_fixations = GazeFeatures.TimeStampedGazeMovements()
     ts_saccades = GazeFeatures.TimeStampedGazeMovements()
+    ts_unknown = GazeFeatures.TimeStampedGazeMovements()
     ts_status = GazeFeatures.TimeStampedGazeStatus()
 
     # Initialise progress bar
-    MiscFeatures.printProgressBar(0, tobii_segment_video.duration, prefix = '\nGazeMovements identification:', suffix = 'Complete', length = 100)
+    #MiscFeatures.printProgressBar(0, tobii_segment_video.duration, prefix = '\nGazeMovements identification:', suffix = 'Complete', length = 100)
 
     for gaze_movement in movement_identifier(ts_gaze_positions):
 
@@ -232,54 +241,84 @@ def main():
             ts_status[end_ts] = GazeFeatures.GazeStatus.from_position(end_position, 'Saccade', len(ts_saccades))
 
         else:
-            continue
+
+            start_ts, start_position = gaze_movement.positions.first
+
+            ts_unknown[start_ts] = gaze_movement
+
+            for ts, position in gaze_movement.positions.items():
+
+                ts_status[ts] = GazeFeatures.GazeStatus.from_position(position, 'UnknownGazeMovement', len(ts_unknown))
 
         # Update Progress Bar
         progress = start_ts - int(args.time_range[0] * 1e6)
-        MiscFeatures.printProgressBar(progress, tobii_segment_video.duration, prefix = 'GazeMovements identification:', suffix = 'Complete', length = 100)
+        #MiscFeatures.printProgressBar(progress, tobii_segment_video.duration, prefix = 'GazeMovements identification:', suffix = 'Complete', length = 100)
 
     print(f'\nGazeMovements identification metrics:')
     print(f'\t{len(ts_fixations)} fixations found')
     print(f'\t{len(ts_saccades)} saccades found')
+    print(f'\t{len(ts_unknown)} unknown movements found')
 
-    # Export fixations analysis
-    fixations_dataframe = ts_fixations.as_dataframe()
-    fixations_dataframe.to_csv(fixations_filepath, index=True)
-    print(f'\nFixations saved into {fixations_filepath}')
+    # Prepare gaze metrics
+    metrics = {}
+    segment_duration = tobii_segment_video.duration * 1e-3
+    metrics['segment_duration (ms)'] = segment_duration
 
-    # Export saccades analysis
-    saccades_dataframe = ts_saccades.as_dataframe()
-    saccades_dataframe.to_csv(saccades_filepath, index=True)
-    print(f'Saccades saved into {saccades_filepath}')
+    fixations_exist = len(ts_fixations) > 0
+    saccades_exist = len(ts_saccades) > 0
+    unknown_exist = len(ts_unknown) > 0
+    status_exist = len(ts_status) > 0
+    
+    # Analyse fixations
+    if fixations_exist:
 
-    # Export gaze status analysis
-    ts_status.as_dataframe().to_csv(gaze_status_filepath, index=True)
-    print(f'Gaze status saved into {gaze_status_filepath}')
+        fixations_dataframe = ts_fixations.as_dataframe()
+        fixations_dataframe.to_csv(fixations_filepath, index=True)
+        print(f'\nFixations saved into {fixations_filepath}')
 
-    # Export gaze metrics
+        exploitation_time = fixations_dataframe.duration.sum() * 1e-3
 
-    # Consider only fixations > duration threshold and saccades < duration threshold
-    # This is mostly useful to filter first and last fixation/saccade as the time range can start anywhere
-    filtered_fixations = fixations_dataframe[fixations_dataframe.duration > args.duration_threshold*1e3]
-    filtered_saccades = saccades_dataframe[saccades_dataframe.duration < args.duration_threshold*1e3]
+        metrics['fixations_number'] = fixations_dataframe.shape[0]
+        metrics['fixations_duration_mean (ms)'] = fixations_dataframe.duration.mean() * 1e-3
+        metrics['exploitation_ratio (%)'] = exploitation_time / segment_duration * 100
 
-    segment_duration = tobii_segment_video.duration * 1e-3
-    exploitation_time = filtered_fixations.duration.sum() * 1e-3
-    exploration_time = filtered_saccades.duration.sum() * 1e-3
-
-    metrics = {
-        'segment_duration (ms)': segment_duration,
-        'fixations_number': filtered_fixations.shape[0],
-        'fixations_duration_mean (ms)': filtered_fixations.duration.mean() * 1e-3, 
-        'saccades_number': filtered_saccades.shape[0],
-        'saccades_duration_mean (ms)': filtered_saccades.duration.mean() * 1e-3,
-        'exploitation_ratio (%)': exploitation_time / segment_duration * 100,
-        'exploration_ratio (%)': exploration_time / segment_duration * 100,
-        'exploit_explore_ratio:': exploitation_time / exploration_time
-        }
+    # Analyse saccades
+    if saccades_exist:
 
-    metrics_dataframe = pandas.DataFrame(metrics, index=[participant_name])
+        saccades_dataframe = ts_saccades.as_dataframe()
+        saccades_dataframe.to_csv(saccades_filepath, index=True)
+        print(f'Saccades saved into {saccades_filepath}')
+
+        exploration_time = saccades_dataframe.duration.sum() * 1e-3
+
+        metrics['saccades_number'] = saccades_dataframe.shape[0]
+        metrics['saccades_duration_mean (ms)'] = saccades_dataframe.duration.mean() * 1e-3
+        metrics['exploration_ratio (%)'] = exploration_time / segment_duration * 100
+    
+    # Export unknown movements analysis
+    if unknown_exist:
+
+        unknown_dataframe = ts_unknown.as_dataframe()
+        unknown_dataframe.to_csv(unknown_filepath, index=True)
+        print(f'Unknown movements saved into {unknown_filepath}')
+
+        unknown_time = unknown_dataframe.duration.sum() * 1e-3
+
+        metrics['unknown_number'] = unknown_dataframe.shape[0]
+        metrics['unknown_duration_mean (ms)'] = unknown_dataframe.duration.mean() * 1e-3
+        metrics['unknown_ratio (%)'] = unknown_time / segment_duration * 100
+
+    if fixations_exist and saccades_exist:
+        metrics['exploit_explore_ratio'] = exploitation_time / exploration_time
+
+    # Export gaze status analysis
+    if status_exist:
+
+        ts_status.as_dataframe().to_csv(gaze_status_filepath, index=True)
+        print(f'Gaze status saved into {gaze_status_filepath}')
 
+    # Export gaze metrics
+    metrics_dataframe = pandas.DataFrame(metrics, index=[participant_name])
     metrics_dataframe.to_csv(gaze_metrics_filepath, index=True)
     print(f'Gaze metrics saved into {gaze_metrics_filepath}')
 
@@ -298,10 +337,15 @@ def main():
         # Initialise progress bar
         MiscFeatures.printProgressBar(0, tobii_segment_video.duration, prefix = '\nGazeMovements visualisation:', suffix = 'Complete', length = 100)
 
-        current_fixation_ts, current_fixation = ts_fixations.pop_first()
-        current_fixation_time_counter = 0
+        if fixations_exist:
+            current_fixation_ts, current_fixation = ts_fixations.pop_first()
+            current_fixation_time_counter = 0
 
-        current_saccade_ts, current_saccade = ts_saccades.pop_first()
+        if saccades_exist:
+            current_saccade_ts, current_saccade = ts_saccades.pop_first()
+
+        if unknown_exist:
+            current_unknown_ts, current_unknown = ts_unknown.pop_first()
 
         # Iterate on video frames
         for video_ts, video_frame in tobii_segment_video.frames():
@@ -315,83 +359,155 @@ def main():
 
                 assert(ts_current_aois == video_ts)
 
-                selected_aoi = AOIFeatures.AreaOfInterest(current_aois.pop(args.aoi))
+                current_aoi = AOIFeatures.AreaOfInterest(current_aois.pop(args.aoi))
 
                 # Apply perspective transform algorithm
                 destination = numpy.float32([[0, 0],[1920, 0],[1920, 1080],[0, 1080]])
-                aoi_matrix = cv.getPerspectiveTransform(selected_aoi.astype(numpy.float32), destination)
+                aoi_matrix = cv.getPerspectiveTransform(current_aoi.astype(numpy.float32), destination)
                 visu_matrix = cv.warpPerspective(video_frame.matrix, aoi_matrix, (1920, 1080))
 
             # Wait for aois projection
             except KeyError:
                 pass
 
-            # Check next fixation
-            if video_ts > current_fixation_ts + current_fixation.duration and len(ts_fixations) > 0:
+            if fixations_exist:
+
+                # Check next fixation
+                if video_ts > current_fixation_ts + current_fixation.duration and len(ts_fixations) > 0:
+
+                    current_fixation_ts, current_fixation = ts_fixations.pop_first()
+                    current_fixation_time_counter = 0
+
+                # While current time belongs to the current fixation
+                if video_ts >= current_fixation_ts and video_ts <= current_fixation_ts + current_fixation.duration:
+
+                    current_fixation_time_counter += 1
+
+                    # Draw current fixation
+                    cv.circle(visu_matrix, (int(current_fixation.centroid[0]), int(current_fixation.centroid[1])), int(current_fixation.dispersion), (0, 255, 0), current_fixation_time_counter)
+                    cv.circle(heatmap_matrix, (int(current_fixation.centroid[0]), int(current_fixation.centroid[1])), int(current_fixation.dispersion), (0, 255, 0))
+
+            if saccades_exist:
+              
+                # Check next saccade
+                if video_ts > current_saccade_ts + current_saccade.duration and len(ts_saccades) > 0:
+
+                    current_saccade_ts, current_saccade = ts_saccades.pop_first()
+
+                # While current time belongs to the current saccade
+                if video_ts >= current_saccade_ts and video_ts <= current_saccade_ts + current_saccade.duration:
+
+                    # Draw all saccade gaze positions
+                    try:
+
+                        # Get next saccade position
+                        ts_next, _ = current_saccade.positions.first
+
+                        # Check next unknown positions is not after current time
+                        while ts_next < video_ts:
+
+                            ts_start, start_gaze_position = current_saccade.positions.pop_first()
+                            ts_next, next_gaze_position = current_saccade.positions.first
 
-                current_fixation_ts, current_fixation = ts_fixations.pop_first()
-                current_fixation_time_counter = 0
+                            # Draw movement
+                            if start_gaze_position.valid and next_gaze_position.valid:
 
-            # While current time belongs to the current fixation
-            if video_ts >= current_fixation_ts and video_ts <= current_fixation_ts + current_fixation.duration:
+                                int_start_position = (int(start_gaze_position[0]), int(start_gaze_position[1]))
+                                int_next_position = (int(next_gaze_position[0]), int(next_gaze_position[1]))
 
-                current_fixation_time_counter += 1
+                                cv.line(visu_matrix, int_start_position, int_next_position, (0, 0, 255), 3)
+                                cv.line(heatmap_matrix, int_start_position, int_next_position, (0, 0, 255), 3)
 
-                # Draw current fixation
-                cv.circle(visu_matrix, (int(current_fixation.centroid[0]), int(current_fixation.centroid[1])), int(current_fixation.dispersion), (0, 255, 0), current_fixation_time_counter)
-                cv.circle(heatmap_matrix, (int(current_fixation.centroid[0]), int(current_fixation.centroid[1])), int(current_fixation.dispersion), (0, 255, 0))
+                    # Empty gaze position
+                    except IndexError:
+                        pass
 
-            # Check next saccade
-            if video_ts > current_saccade_ts + current_saccade.duration and len(ts_saccades) > 0:
+            if unknown_exist:
 
-                current_saccade_ts, current_saccade = ts_saccades.pop_first()
+                # Check next unknown movement
+                if video_ts > current_unknown_ts + current_unknown.duration and len(ts_unknown) > 0:
 
-            # While current time belongs to the current saccade
-            if video_ts >= current_saccade_ts and video_ts <= current_saccade_ts + current_saccade.duration:
+                    current_unknown_ts, current_unknown = ts_unknown.pop_first()
 
-                start_ts, start_position = current_saccade.positions.first
-                end_ts, end_position = current_saccade.positions.last
+                # While current time belongs to the current unknown movement
+                if video_ts >= current_unknown_ts and video_ts <= current_unknown_ts + current_unknown.duration:
 
-                # Draw saccade
-                int_start_position = (int(start_position[0]), int(start_position[1]))
-                int_end_position = (int(end_position[0]), int(end_position[1]))
+                    # Draw all unknown gaze positions
+                    try:
 
-                cv.line(visu_matrix, int_start_position, int_end_position, (0, 0, 255), 2)
-                cv.line(heatmap_matrix, int_start_position, int_end_position, (0, 0, 255), 2)
+                        # Get next unknown position
+                        ts_next, _ = current_unknown.positions.first
 
-                # Write start gaze position
-                cv.putText(visu_matrix, str(int_start_position), int_start_position, cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 1, cv.LINE_AA)
-                cv.putText(visu_matrix, str(int_end_position), int_end_position, cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 1, cv.LINE_AA)
+                        # Check next unknown positions is not after current time
+                        while ts_next < video_ts:
 
-            # Check next gaze
+                            ts_start, start_gaze_position = current_unknown.positions.pop_first()
+                            ts_next, next_gaze_position = current_unknown.positions.first
+                            
+                            # Draw movement
+                            if start_gaze_position.valid and next_gaze_position.valid:
+
+                                int_start_position = (int(start_gaze_position[0]), int(start_gaze_position[1]))
+                                int_next_position = (int(next_gaze_position[0]), int(next_gaze_position[1]))
+
+                                cv.line(visu_matrix, int_start_position, int_next_position, (255, 0, 0), 3)
+                                cv.line(heatmap_matrix, int_start_position, int_next_position, (255, 0, 0), 3)
+
+                    # Empty gaze position
+                    except IndexError:
+                        pass
+
+            # Draw all next gaze positions
             try:
 
-                # Get closest gaze position before video timestamp and remove all gaze positions before
-                ts_nearest, nearest_gaze_position = ts_gaze_positions.pop_last_until(video_ts)
+                # Get next gaze position
+                ts_next, next_gaze_position = ts_gaze_positions.first
 
-                # Draw gaze
-                nearest_gaze_position.draw(visu_matrix)
-                nearest_gaze_position.draw(heatmap_matrix)
+                # Check next gaze position is not after current time
+                while ts_next < video_ts:
 
-                # Write gaze position
-                cv.putText(visu_matrix, str(nearest_gaze_position.value), nearest_gaze_position.value, cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
-                
-                # Write gaze timing
-                cv.rectangle(visu_matrix, (0, 50), (550, 100), (31, 31, 31), -1)
-                cv.putText(visu_matrix, f'Gaze time: {ts_nearest*1e-3:.3f} ms', (20, 85), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
+                    ts_start, start_gaze_position = ts_gaze_positions.pop_first()
+                    ts_next, next_gaze_position = ts_gaze_positions.first
+
+                    # Draw movement
+                    if start_gaze_position.valid and next_gaze_position.valid:
+
+                        int_start_position = (int(start_gaze_position[0]), int(start_gaze_position[1]))
+                        int_next_position = (int(next_gaze_position[0]), int(next_gaze_position[1]))
+
+                        cv.line(visu_matrix, int_start_position, int_next_position, (0, 255, 255), 1)
+                        cv.line(heatmap_matrix, int_start_position, int_next_position, (0, 255, 255), 1)
+
+                    # Draw gaze
+                    next_gaze_position.draw(visu_matrix)
+                    next_gaze_position.draw(heatmap_matrix)
+
+                # Write last gaze position
+                if next_gaze_position.valid:
+
+                    int_next_position = (int(next_gaze_position[0]), int(next_gaze_position[1]))
+                    cv.putText(visu_matrix, str(int_next_position), int_next_position, cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv.LINE_AA)
             
-            # Wait for gaze position
-            except KeyError:
+            # Empty gaze position
+            except IndexError:
                 pass
-
+            
+            # Write last gaze position timing
+            cv.rectangle(visu_matrix, (0, 50), (550, 100), (31, 31, 31), -1)
+            cv.putText(visu_matrix, f'Gaze time: {ts_next*1e-3:.3f} ms', (20, 85), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+        
             # Write segment timing
             cv.rectangle(visu_matrix, (0, 0), (550, 50), (63, 63, 63), -1)
             cv.putText(visu_matrix, f'Video time: {video_ts*1e-3:.3f} ms', (20, 40), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
             
             # Write movement identification parameters
-            cv.rectangle(visu_matrix, (0, 100), (550, 200), (63, 63, 63), -1)
-            cv.putText(visu_matrix, f'Dispersion threshold: {args.dispersion_threshold} px', (20, 140), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
-            cv.putText(visu_matrix, f'Duration threshold: {args.duration_threshold} ms', (20, 190), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+            cv.rectangle(visu_matrix, (0, 100), (550, 260), (63, 63, 63), -1)
+            cv.putText(visu_matrix, f'Dispersion max: {args.dispersion_threshold} px', (20, 160), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+            cv.putText(visu_matrix, f'Duration min: {args.duration_threshold} ms', (20, 220), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+
+            # Draw dispersion threshold circle
+            cv.circle(visu_matrix, (args.dispersion_threshold + 400, 180), 2, (0, 255, 255), -1)
+            cv.circle(visu_matrix, (args.dispersion_threshold + 400, 180), args.dispersion_threshold, (255, 150, 150), 1)
 
             if args.window: