Exporting new movements.csv and movements.mp4 files.

3 files changed, 166 insertions, 30 deletions

diff --git a/src/argaze/GazeFeatures.py b/src/argaze/GazeFeatures.py
index 32ec571..d7466df 100644
--- a/src/argaze/GazeFeatures.py
+++ b/src/argaze/GazeFeatures.py
@@ -22,6 +22,7 @@ class TimeStampedGazePositions(DataStructures.TimeStampedBuffer):
 class Fixation():
     """Define gaze fixation."""
 
+    index: int
     duration: float
     dispersion: float
     centroid: GazePosition
@@ -31,12 +32,18 @@ class TimeStampedFixations(DataStructures.TimeStampedBuffer):
     """Define timestamped buffer to store fixations."""
 
     def __setitem__(self, key, value: Fixation):
+        """Force value to be a Fixation"""
+
+        if not isinstance(value, Fixation):
+            raise ValueError('value must be a Fixation')
+
         super().__setitem__(key, value)
 
 @dataclass
 class Saccade():
     """Define gaze saccade."""
 
+    index: int
     duration: float
     start_position: GazePosition
     end_position: GazePosition
@@ -46,11 +53,31 @@ class TimeStampedSaccades(DataStructures.TimeStampedBuffer):
 
     def __setitem__(self, key, value: Saccade):
         """Force value to be a Saccade"""
+
         if not isinstance(value, Saccade):
             raise ValueError('value must be a Saccade')
 
         super().__setitem__(key, value)
 
+@dataclass
+class Movement():
+    """Define movement."""
+
+    index: int
+    type: str
+    position: GazePosition
+
+class TimeStampedMovements(DataStructures.TimeStampedBuffer):
+    """Define timestamped buffer to store movement."""
+
+    def __setitem__(self, key, value: Movement):
+        """Force value to be a Movement"""
+        
+        if not isinstance(value, Movement):
+            raise ValueError('value must be a Movement')
+
+        super().__setitem__(key, value)
+
 class MovementIdentifier():
     """Abstract class to define what should provide a movement identifier."""
 
@@ -67,21 +94,31 @@ class MovementIdentifier():
 
     def identify(self):
 
-        fixations = GazeFeatures.TimeStampedFixations()
-        saccades = GazeFeatures.TimeStampedSaccades()
+        fixations = TimeStampedFixations()
+        saccades = TimeStampedSaccades()
+        movements = TimeStampedMovement()
 
         for ts, item in self:
 
             if isinstance(item, GazeFeatures.Fixation):
+
                 fixations[ts] = item
 
+                for ts, position in item.positions.items():
+
+                    movements[ts] = Movement(item.index, type(item).__name__, position)
+
             elif isinstance(item, GazeFeatures.Saccade):
+
                 saccades[ts] = item
 
+                movements[ts] = Movement(item.index, type(item).__name__, item.start_position)
+                movements[ts + item.duration] = Movement(item.index, type(item).__name__, item.end_position)
+
             else:
                 continue
 
-        return fixations, saccades
+        return fixations, saccades, movements
 
 class DispersionBasedMovementIdentifier(MovementIdentifier):
     """Implementation of the I-DT algorithm as described in:
@@ -105,6 +142,9 @@ class DispersionBasedMovementIdentifier(MovementIdentifier):
         self.__last_fixation = None
         self.__last_fixation_ts = -1
 
+        self.__fixations_count = 0
+        self.__saccades_count = 0
+
     def __getEuclideanDispersion(self, ts_gaze_positions_list):
         """Euclidian dispersion algorithm"""
 
@@ -201,7 +241,9 @@ class DispersionBasedMovementIdentifier(MovementIdentifier):
                     for (ts, gp) in ts_gaze_positions_list:
                         ts_gaze_positions[round(ts)] = gp
 
-                    new_fixation = Fixation(round(duration), dispersion, (round(cx), round(cy)), ts_gaze_positions)
+                    self.__fixations_count += 1
+
+                    new_fixation = Fixation(self.__fixations_count, round(duration), dispersion, (round(cx), round(cy)), ts_gaze_positions)
                     new_fixation_ts = ts_list[0]
 
                     if self.__last_fixation != None:
@@ -211,7 +253,12 @@ class DispersionBasedMovementIdentifier(MovementIdentifier):
 
                         if new_saccade_duration > 0:
 
-                            new_saccade = Saccade(round(new_saccade_duration), self.__last_fixation.positions.pop_last()[1], new_fixation.positions.pop_first()[1])
+                            start_position_ts, start_position = self.__last_fixation.positions.pop_last()
+                            end_position_ts, end_position = new_fixation.positions.pop_first()
+
+                            self.__saccades_count += 1
+                            
+                            new_saccade = Saccade(self.__saccades_count, round(new_saccade_duration), start_position, end_position)
                         
                             yield round(new_saccade_ts), new_saccade
 
diff --git a/src/argaze/TobiiGlassesPro2/TobiiController.py b/src/argaze/TobiiGlassesPro2/TobiiController.py
index c77738d..7be289f 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiController.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiController.py
@@ -269,6 +269,21 @@ class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface):
 	def get_recordings(self):
 		return super().get_request('/api/recordings')
 
+	# EVENT AND VARIABLES
+
+	def send_custom_event(self, event_type, event_tag = ''):
+		data = {'type': event_type, 'tag': event_tag}
+		super().post_request('/api/events', data, wait_for_response=False)
+
+	def send_tobiipro_event(self, event_type, event_value):
+		self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value))
+
+	def send_experimental_var(self, variable_name, variable_value):
+		self.send_custom_event('#%s#' % variable_name, variable_value)
+
+	def send_experimental_vars(self, variable_names_list, variable_values_list):
+		self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list))
+
 	# MISC
 
 	def eject_sd(self):
@@ -316,19 +331,6 @@ class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface):
 	def get_video_freq(self):
 		return self.get_configuration()['sys_sc_fps']
 
-	def send_custom_event(self, event_type, event_tag = ''):
-		data = {'type': event_type, 'tag': event_tag}
-		super().post_request('/api/events', data, wait_for_response=False)
-
-	def send_experimental_var(self, variable_name, variable_value):
-		self.send_custom_event('#%s#' % variable_name, variable_value)
-
-	def send_experimental_vars(self, variable_names_list, variable_values_list):
-		self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list))
-
-	def send_tobiipro_event(self, event_type, event_value):
-		self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value))
-
 	def set_et_freq_50(self):
 		data = {'sys_et_freq': 50}
 		json_data = super().post_request('/api/system/conf', data)
diff --git a/src/argaze/utils/export_tobii_segment_movements.py b/src/argaze/utils/export_tobii_segment_movements.py
index ad06312..a36fe58 100644
--- a/src/argaze/utils/export_tobii_segment_movements.py
+++ b/src/argaze/utils/export_tobii_segment_movements.py
@@ -4,9 +4,11 @@ import argparse
 import os
 
 from argaze import GazeFeatures
-from argaze.TobiiGlassesPro2 import TobiiEntities
+from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo
 from argaze.utils import MiscFeatures
 
+import cv2 as cv
+
 def main():
     """
     Analyse Tobii segment fixations
@@ -31,13 +33,17 @@ def main():
                 os.makedirs(os.path.dirname(args.output))
                 print(f'{os.path.dirname(args.output)} folder created')
 
-            fixations_filepath = f'{args.output}/fixations.csv'
-            saccades_filepath = f'{args.output}/saccades.csv'
+            gaze_video_filepath = f'{args.output}/movements.mp4'
+            fixations_filepath = f'{args.output}/movements_fixations.csv'
+            saccades_filepath = f'{args.output}/movements_saccades.csv'
+            movements_filepath = f'{args.output}/movements.csv'
 
         else:
 
-            fixations_filepath = f'{args.segment_path}/fixations.csv'
-            saccades_filepath = f'{args.segment_path}/saccades.csv'
+            gaze_video_filepath = f'{args.segment_path}/movements.mp4'
+            fixations_filepath = f'{args.segment_path}/movements_fixations.csv'
+            saccades_filepath = f'{args.segment_path}/movements_saccades.csv'
+            movements_filepath = f'{args.segment_path}/movements.csv'
 
         # Load a tobii segment
         tobii_segment = TobiiEntities.TobiiSegment(args.segment_path, int(args.time_range[0] * 1000000), int(args.time_range[1] * 1000000) if args.time_range[1] != None else None)
@@ -54,38 +60,48 @@ def main():
         tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp
         print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded')
 
-        # Format tobii gaze data into generic gaze data and store them using millisecond unit timestamp
-        generic_ts_gaze_positions = GazeFeatures.TimeStampedGazePositions()
+        # Format tobii gaze position in pixel and store them using millisecond unit timestamp
+        ts_gaze_positions = GazeFeatures.TimeStampedGazePositions()
 
         for ts, tobii_data in tobii_ts_gaze_positions.items():
-            generic_data = (int(tobii_data.gp[0] * tobii_segment_video.get_width()), int(tobii_data.gp[1] * tobii_segment_video.get_height()))
-            generic_ts_gaze_positions[ts/1000] = generic_data
+            video_gaze_pixel = (int(tobii_data.gp[0] * tobii_segment_video.get_width()), int(tobii_data.gp[1] * tobii_segment_video.get_height()))
+            ts_gaze_positions[ts/1000] = video_gaze_pixel
 
         print(f'Dispersion threshold: {args.dispersion_threshold}')
         print(f'Duration threshold: {args.duration_threshold}')
 
         # Start movement identification
-        movement_identifier = GazeFeatures.DispersionBasedMovementIdentifier(generic_ts_gaze_positions, args.dispersion_threshold, args.duration_threshold)
+        movement_identifier = GazeFeatures.DispersionBasedMovementIdentifier(ts_gaze_positions, args.dispersion_threshold, args.duration_threshold)
         fixations = GazeFeatures.TimeStampedFixations()
         saccades = GazeFeatures.TimeStampedSaccades()
+        movements = GazeFeatures.TimeStampedMovements()
 
         # Initialise progress bar
-        MiscFeatures.printProgressBar(0, int(tobii_segment_video.get_duration()/1000), prefix = 'Progress:', suffix = 'Complete', length = 100)
+        MiscFeatures.printProgressBar(0, int(tobii_segment_video.get_duration()/1000), prefix = 'Movements identification:', suffix = 'Complete', length = 100)
 
         for ts, item in movement_identifier:
 
             if isinstance(item, GazeFeatures.Fixation):
+
                 fixations[ts] = item
 
+                for ts, position in item.positions.items():
+
+                    movements[ts] = GazeFeatures.Movement(item.index, type(item).__name__, position)
+
             elif isinstance(item, GazeFeatures.Saccade):
+
                 saccades[ts] = item
 
+                movements[ts] = GazeFeatures.Movement(item.index, type(item).__name__, item.start_position)
+                movements[ts + item.duration] = GazeFeatures.Movement(item.index, type(item).__name__, item.end_position)
+
             else:
                 continue
 
             # Update Progress Bar
             progress = ts - int(args.time_range[0] * 1000)
-            MiscFeatures.printProgressBar(progress, int(tobii_segment_video.get_duration()/1000), prefix = 'Progress:', suffix = 'Complete', length = 100)
+            MiscFeatures.printProgressBar(progress, int(tobii_segment_video.get_duration()/1000), prefix = 'Movements identification:', suffix = 'Complete', length = 100)
 
         print(f'\n{len(fixations)} fixations and {len(saccades)} saccades found')
 
@@ -97,6 +113,77 @@ def main():
         saccades.export_as_csv(saccades_filepath)
         print(f'Saccades saved into {saccades_filepath}')
 
+        # Export movements analysis
+        movements.export_as_csv(movements_filepath)
+        print(f'Movements saved into {movements_filepath}')
+
+        # Prepare video exportation at the same format than segment video
+        output_video = TobiiVideo.TobiiVideoOutput(gaze_video_filepath, tobii_segment_video.get_stream())
+
+        # Video and data loop
+        try:
+
+            # Initialise progress bar
+            MiscFeatures.printProgressBar(0, tobii_segment_video.get_duration()/1000, prefix = 'Video with movements processing:', suffix = 'Complete', length = 100)
+
+            current_fixation_ts, current_fixation = fixations.pop_first()
+            time_counter = 0
+
+            # Iterate on video frames
+            for video_ts, video_frame in tobii_segment_video.frames():
+
+                video_ts_ms = video_ts / 1000
+
+                # write segment timing
+                cv.putText(video_frame.matrix, f'Segment time: {int(video_ts_ms)} ms', (20, 40), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+
+                # write movement identification parameters
+                cv.putText(video_frame.matrix, f'Dispersion threshold: {args.dispersion_threshold} px', (20, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+                cv.putText(video_frame.matrix, f'Duration threshold: {args.duration_threshold} ms', (20, 140), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA)
+
+                if len(fixations) > 0:
+
+                    if video_ts_ms > current_fixation_ts + current_fixation.duration:
+
+                        current_fixation_ts, current_fixation = fixations.pop_first()
+                        time_counter = 0
+
+                    else:
+
+                        time_counter += 1
+
+                    # Draw current fixation
+                    cv.circle(video_frame.matrix, current_fixation.centroid, current_fixation.dispersion + time_counter, (0, 255, 255), 1)
+
+                try:
+
+                    # Get closest gaze position before video timestamp and remove all gaze positions before
+                    closest_gaze_ts, closest_gaze_position = ts_gaze_positions.pop_first_until(video_ts_ms)
+
+                    # Draw gaze position
+                    cv.circle(video_frame.matrix, closest_gaze_position, 10, (0, 255, 255), 2)
+
+                # Wait for gaze position
+                except ValueError:
+                    pass
+
+                # Write video
+                output_video.write(video_frame.matrix)
+
+                # Update Progress Bar
+                progress = video_ts_ms - int(args.time_range[0] * 1000)
+                MiscFeatures.printProgressBar(progress, tobii_segment_video.get_duration()/1000, prefix = 'Video with movements processing:', suffix = 'Complete', length = 100)
+
+        # Exit on 'ctrl+C' interruption
+        except KeyboardInterrupt:
+            pass
+
+        # End output video file
+        output_video.close()
+        print(f'\nVideo with movements saved into {gaze_video_filepath}')
+
+        
+
 if __name__ == '__main__':
 
     main()
\ No newline at end of file