From f59fae48f03fc29b315b9ea750c01e147697e3ff Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Tue, 10 May 2022 11:19:43 +0200 Subject: Exporting new movements.csv and movements.mp4 files. --- src/argaze/GazeFeatures.py | 57 ++++++++++- src/argaze/TobiiGlassesPro2/TobiiController.py | 28 +++--- src/argaze/utils/export_tobii_segment_movements.py | 111 ++++++++++++++++++--- 3 files changed, 166 insertions(+), 30 deletions(-) diff --git a/src/argaze/GazeFeatures.py b/src/argaze/GazeFeatures.py index 32ec571..d7466df 100644 --- a/src/argaze/GazeFeatures.py +++ b/src/argaze/GazeFeatures.py @@ -22,6 +22,7 @@ class TimeStampedGazePositions(DataStructures.TimeStampedBuffer): class Fixation(): """Define gaze fixation.""" + index: int duration: float dispersion: float centroid: GazePosition @@ -31,12 +32,18 @@ class TimeStampedFixations(DataStructures.TimeStampedBuffer): """Define timestamped buffer to store fixations.""" def __setitem__(self, key, value: Fixation): + """Force value to be a Fixation""" + + if not isinstance(value, Fixation): + raise ValueError('value must be a Fixation') + super().__setitem__(key, value) @dataclass class Saccade(): """Define gaze saccade.""" + index: int duration: float start_position: GazePosition end_position: GazePosition @@ -46,11 +53,31 @@ class TimeStampedSaccades(DataStructures.TimeStampedBuffer): def __setitem__(self, key, value: Saccade): """Force value to be a Saccade""" + if not isinstance(value, Saccade): raise ValueError('value must be a Saccade') super().__setitem__(key, value) +@dataclass +class Movement(): + """Define movement.""" + + index: int + type: str + position: GazePosition + +class TimeStampedMovements(DataStructures.TimeStampedBuffer): + """Define timestamped buffer to store movement.""" + + def __setitem__(self, key, value: Movement): + """Force value to be a Movement""" + + if not isinstance(value, Movement): + raise ValueError('value must be a Movement') + + super().__setitem__(key, value) + class MovementIdentifier(): """Abstract class to define what should provide a movement identifier.""" @@ -67,21 +94,31 @@ class MovementIdentifier(): def identify(self): - fixations = GazeFeatures.TimeStampedFixations() - saccades = GazeFeatures.TimeStampedSaccades() + fixations = TimeStampedFixations() + saccades = TimeStampedSaccades() + movements = TimeStampedMovement() for ts, item in self: if isinstance(item, GazeFeatures.Fixation): + fixations[ts] = item + for ts, position in item.positions.items(): + + movements[ts] = Movement(item.index, type(item).__name__, position) + elif isinstance(item, GazeFeatures.Saccade): + saccades[ts] = item + movements[ts] = Movement(item.index, type(item).__name__, item.start_position) + movements[ts + item.duration] = Movement(item.index, type(item).__name__, item.end_position) + else: continue - return fixations, saccades + return fixations, saccades, movements class DispersionBasedMovementIdentifier(MovementIdentifier): """Implementation of the I-DT algorithm as described in: @@ -105,6 +142,9 @@ class DispersionBasedMovementIdentifier(MovementIdentifier): self.__last_fixation = None self.__last_fixation_ts = -1 + self.__fixations_count = 0 + self.__saccades_count = 0 + def __getEuclideanDispersion(self, ts_gaze_positions_list): """Euclidian dispersion algorithm""" @@ -201,7 +241,9 @@ class DispersionBasedMovementIdentifier(MovementIdentifier): for (ts, gp) in ts_gaze_positions_list: ts_gaze_positions[round(ts)] = gp - new_fixation = Fixation(round(duration), dispersion, (round(cx), round(cy)), ts_gaze_positions) + self.__fixations_count += 1 + + new_fixation = Fixation(self.__fixations_count, round(duration), dispersion, (round(cx), round(cy)), ts_gaze_positions) new_fixation_ts = ts_list[0] if self.__last_fixation != None: @@ -211,7 +253,12 @@ class DispersionBasedMovementIdentifier(MovementIdentifier): if new_saccade_duration > 0: - new_saccade = Saccade(round(new_saccade_duration), self.__last_fixation.positions.pop_last()[1], new_fixation.positions.pop_first()[1]) + start_position_ts, start_position = self.__last_fixation.positions.pop_last() + end_position_ts, end_position = new_fixation.positions.pop_first() + + self.__saccades_count += 1 + + new_saccade = Saccade(self.__saccades_count, round(new_saccade_duration), start_position, end_position) yield round(new_saccade_ts), new_saccade diff --git a/src/argaze/TobiiGlassesPro2/TobiiController.py b/src/argaze/TobiiGlassesPro2/TobiiController.py index c77738d..7be289f 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiController.py +++ b/src/argaze/TobiiGlassesPro2/TobiiController.py @@ -269,6 +269,21 @@ class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface): def get_recordings(self): return super().get_request('/api/recordings') + # EVENT AND VARIABLES + + def send_custom_event(self, event_type, event_tag = ''): + data = {'type': event_type, 'tag': event_tag} + super().post_request('/api/events', data, wait_for_response=False) + + def send_tobiipro_event(self, event_type, event_value): + self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value)) + + def send_experimental_var(self, variable_name, variable_value): + self.send_custom_event('#%s#' % variable_name, variable_value) + + def send_experimental_vars(self, variable_names_list, variable_values_list): + self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list)) + # MISC def eject_sd(self): @@ -316,19 +331,6 @@ class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface): def get_video_freq(self): return self.get_configuration()['sys_sc_fps'] - def send_custom_event(self, event_type, event_tag = ''): - data = {'type': event_type, 'tag': event_tag} - super().post_request('/api/events', data, wait_for_response=False) - - def send_experimental_var(self, variable_name, variable_value): - self.send_custom_event('#%s#' % variable_name, variable_value) - - def send_experimental_vars(self, variable_names_list, variable_values_list): - self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list)) - - def send_tobiipro_event(self, event_type, event_value): - self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value)) - def set_et_freq_50(self): data = {'sys_et_freq': 50} json_data = super().post_request('/api/system/conf', data) diff --git a/src/argaze/utils/export_tobii_segment_movements.py b/src/argaze/utils/export_tobii_segment_movements.py index ad06312..a36fe58 100644 --- a/src/argaze/utils/export_tobii_segment_movements.py +++ b/src/argaze/utils/export_tobii_segment_movements.py @@ -4,9 +4,11 @@ import argparse import os from argaze import GazeFeatures -from argaze.TobiiGlassesPro2 import TobiiEntities +from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo from argaze.utils import MiscFeatures +import cv2 as cv + def main(): """ Analyse Tobii segment fixations @@ -31,13 +33,17 @@ def main(): os.makedirs(os.path.dirname(args.output)) print(f'{os.path.dirname(args.output)} folder created') - fixations_filepath = f'{args.output}/fixations.csv' - saccades_filepath = f'{args.output}/saccades.csv' + gaze_video_filepath = f'{args.output}/movements.mp4' + fixations_filepath = f'{args.output}/movements_fixations.csv' + saccades_filepath = f'{args.output}/movements_saccades.csv' + movements_filepath = f'{args.output}/movements.csv' else: - fixations_filepath = f'{args.segment_path}/fixations.csv' - saccades_filepath = f'{args.segment_path}/saccades.csv' + gaze_video_filepath = f'{args.segment_path}/movements.mp4' + fixations_filepath = f'{args.segment_path}/movements_fixations.csv' + saccades_filepath = f'{args.segment_path}/movements_saccades.csv' + movements_filepath = f'{args.segment_path}/movements.csv' # Load a tobii segment tobii_segment = TobiiEntities.TobiiSegment(args.segment_path, int(args.time_range[0] * 1000000), int(args.time_range[1] * 1000000) if args.time_range[1] != None else None) @@ -54,38 +60,48 @@ def main(): tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded') - # Format tobii gaze data into generic gaze data and store them using millisecond unit timestamp - generic_ts_gaze_positions = GazeFeatures.TimeStampedGazePositions() + # Format tobii gaze position in pixel and store them using millisecond unit timestamp + ts_gaze_positions = GazeFeatures.TimeStampedGazePositions() for ts, tobii_data in tobii_ts_gaze_positions.items(): - generic_data = (int(tobii_data.gp[0] * tobii_segment_video.get_width()), int(tobii_data.gp[1] * tobii_segment_video.get_height())) - generic_ts_gaze_positions[ts/1000] = generic_data + video_gaze_pixel = (int(tobii_data.gp[0] * tobii_segment_video.get_width()), int(tobii_data.gp[1] * tobii_segment_video.get_height())) + ts_gaze_positions[ts/1000] = video_gaze_pixel print(f'Dispersion threshold: {args.dispersion_threshold}') print(f'Duration threshold: {args.duration_threshold}') # Start movement identification - movement_identifier = GazeFeatures.DispersionBasedMovementIdentifier(generic_ts_gaze_positions, args.dispersion_threshold, args.duration_threshold) + movement_identifier = GazeFeatures.DispersionBasedMovementIdentifier(ts_gaze_positions, args.dispersion_threshold, args.duration_threshold) fixations = GazeFeatures.TimeStampedFixations() saccades = GazeFeatures.TimeStampedSaccades() + movements = GazeFeatures.TimeStampedMovements() # Initialise progress bar - MiscFeatures.printProgressBar(0, int(tobii_segment_video.get_duration()/1000), prefix = 'Progress:', suffix = 'Complete', length = 100) + MiscFeatures.printProgressBar(0, int(tobii_segment_video.get_duration()/1000), prefix = 'Movements identification:', suffix = 'Complete', length = 100) for ts, item in movement_identifier: if isinstance(item, GazeFeatures.Fixation): + fixations[ts] = item + for ts, position in item.positions.items(): + + movements[ts] = GazeFeatures.Movement(item.index, type(item).__name__, position) + elif isinstance(item, GazeFeatures.Saccade): + saccades[ts] = item + movements[ts] = GazeFeatures.Movement(item.index, type(item).__name__, item.start_position) + movements[ts + item.duration] = GazeFeatures.Movement(item.index, type(item).__name__, item.end_position) + else: continue # Update Progress Bar progress = ts - int(args.time_range[0] * 1000) - MiscFeatures.printProgressBar(progress, int(tobii_segment_video.get_duration()/1000), prefix = 'Progress:', suffix = 'Complete', length = 100) + MiscFeatures.printProgressBar(progress, int(tobii_segment_video.get_duration()/1000), prefix = 'Movements identification:', suffix = 'Complete', length = 100) print(f'\n{len(fixations)} fixations and {len(saccades)} saccades found') @@ -97,6 +113,77 @@ def main(): saccades.export_as_csv(saccades_filepath) print(f'Saccades saved into {saccades_filepath}') + # Export movements analysis + movements.export_as_csv(movements_filepath) + print(f'Movements saved into {movements_filepath}') + + # Prepare video exportation at the same format than segment video + output_video = TobiiVideo.TobiiVideoOutput(gaze_video_filepath, tobii_segment_video.get_stream()) + + # Video and data loop + try: + + # Initialise progress bar + MiscFeatures.printProgressBar(0, tobii_segment_video.get_duration()/1000, prefix = 'Video with movements processing:', suffix = 'Complete', length = 100) + + current_fixation_ts, current_fixation = fixations.pop_first() + time_counter = 0 + + # Iterate on video frames + for video_ts, video_frame in tobii_segment_video.frames(): + + video_ts_ms = video_ts / 1000 + + # write segment timing + cv.putText(video_frame.matrix, f'Segment time: {int(video_ts_ms)} ms', (20, 40), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA) + + # write movement identification parameters + cv.putText(video_frame.matrix, f'Dispersion threshold: {args.dispersion_threshold} px', (20, 100), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA) + cv.putText(video_frame.matrix, f'Duration threshold: {args.duration_threshold} ms', (20, 140), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA) + + if len(fixations) > 0: + + if video_ts_ms > current_fixation_ts + current_fixation.duration: + + current_fixation_ts, current_fixation = fixations.pop_first() + time_counter = 0 + + else: + + time_counter += 1 + + # Draw current fixation + cv.circle(video_frame.matrix, current_fixation.centroid, current_fixation.dispersion + time_counter, (0, 255, 255), 1) + + try: + + # Get closest gaze position before video timestamp and remove all gaze positions before + closest_gaze_ts, closest_gaze_position = ts_gaze_positions.pop_first_until(video_ts_ms) + + # Draw gaze position + cv.circle(video_frame.matrix, closest_gaze_position, 10, (0, 255, 255), 2) + + # Wait for gaze position + except ValueError: + pass + + # Write video + output_video.write(video_frame.matrix) + + # Update Progress Bar + progress = video_ts_ms - int(args.time_range[0] * 1000) + MiscFeatures.printProgressBar(progress, tobii_segment_video.get_duration()/1000, prefix = 'Video with movements processing:', suffix = 'Complete', length = 100) + + # Exit on 'ctrl+C' interruption + except KeyboardInterrupt: + pass + + # End output video file + output_video.close() + print(f'\nVideo with movements saved into {gaze_video_filepath}') + + + if __name__ == '__main__': main() \ No newline at end of file -- cgit v1.1