From c98e3baa548791d7a627b37f287915a3f30585d1 Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Thu, 18 Jan 2024 13:12:39 +0100 Subject: First work on TimestampedDataLogger system. Replacing log attribute by loggers dictionary in ArFrame and ArLayer. --- .../advanced_topics/scripting.md | 32 +++-- src/argaze.test/DataLog/File.py | 37 ++++++ src/argaze.test/DataLog/__init__.py | 0 src/argaze/ArFeatures.py | 140 ++++++++++----------- src/argaze/DataFeatures.py | 8 ++ src/argaze/DataLog/File.py | 47 +++++++ src/argaze/DataLog/__init__.py | 4 + .../utils/demo_data/demo_gaze_analysis_setup.json | 12 ++ 8 files changed, 200 insertions(+), 80 deletions(-) create mode 100644 src/argaze.test/DataLog/File.py create mode 100644 src/argaze.test/DataLog/__init__.py create mode 100644 src/argaze/DataLog/File.py create mode 100644 src/argaze/DataLog/__init__.py diff --git a/docs/user_guide/gaze_analysis_pipeline/advanced_topics/scripting.md b/docs/user_guide/gaze_analysis_pipeline/advanced_topics/scripting.md index 4e2be92..ab0c6d0 100644 --- a/docs/user_guide/gaze_analysis_pipeline/advanced_topics/scripting.md +++ b/docs/user_guide/gaze_analysis_pipeline/advanced_topics/scripting.md @@ -72,7 +72,7 @@ for name, ar_layer in ar_frame.layers.items(): ... # Look ArFrame at a timestamped gaze position - gaze_position, gaze_movement, scan_path_analysis, layers_analysis, execution_times, exception = ar_frame.look(timestamp, gaze_position) + gaze_position, gaze_movement, scan_path_analysis, execution_times, exception, layers_look_data = ar_frame.look(timestamp, gaze_position) # Check if a gaze movement has been identified if gaze_movement.valid and gaze_movement.finished: @@ -90,18 +90,32 @@ for name, ar_layer in ar_frame.layers.items(): for data, value in analysis.items(): ... - # Do something with each layer AOI scan path analysis - for layer_name, layer_aoi_scan_path_analysis in layers_analysis.items(): - for module, analysis in layer_aoi_scan_path_analysis.items(): + # Do something with ArFrame look execution times + ... + + # Do something with ArFrame look exception + if exception: + ... + + # Do something with each ArLayer look data + for layer_name, layer_look_data in layers_look_data.items(): + + looked_aoi_name, aoi_scan_path_analysis, layer_execution_times, layer_exception = layer_look_data + + # Do something with looked AOI name + ... + + # Do something with ArLayer AOI scan path analysis + for module, analysis in aoi_scan_path_analysis.items(): for data, value in analysis.items(): ... - # Do something with pipeline execution times - ... + # Do something with ArLayer look execution times + ... - # Do something with pipeline exception - if exception: - ... + # Do something with ArLayer look exception + if exception: + ... ``` Let's understand the meaning of each returned data. diff --git a/src/argaze.test/DataLog/File.py b/src/argaze.test/DataLog/File.py new file mode 100644 index 0000000..2d66ea6 --- /dev/null +++ b/src/argaze.test/DataLog/File.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +""" """ + +__author__ = "Théo de la Hogue" +__credits__ = [] +__copyright__ = "Copyright 2023, Ecole Nationale de l'Aviation Civile (ENAC)" +__license__ = "BSD" + +import unittest +import os + +from argaze import DataFeatures +from argaze.DataLog import File +from argaze.utils import UtilsFeatures + +DataFeaturesTest = UtilsFeatures.importFromTestPackage('DataFeatures') + +class TestTimeStampedDataLogger(unittest.TestCase): + """Test DataLogger class.""" + + def test_creation(self): + """Test logger creation.""" + + file_logger = File.TimeStampedDataLogger(path='./_export/logs/data.txt', separator=',') + + # Check file creation + self.assertEqual(os.path.exists('./_export/logs/data.txt'), True) + + # Write into file + file_logger.emit(0, 'A') + file_logger.emit(1, 'B') + file_logger.emit(2, 'C') + +if __name__ == '__main__': + + unittest.main() \ No newline at end of file diff --git a/src/argaze.test/DataLog/__init__.py b/src/argaze.test/DataLog/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/argaze/ArFeatures.py b/src/argaze/ArFeatures.py index 7263b94..f90354a 100644 --- a/src/argaze/ArFeatures.py +++ b/src/argaze/ArFeatures.py @@ -116,7 +116,7 @@ class ArLayer(DataFeatures.SharedObject): aoi_matcher: GazeFeatures.AOIMatcher = field(default_factory=GazeFeatures.AOIMatcher) aoi_scan_path: GazeFeatures.AOIScanPath = field(default_factory=GazeFeatures.AOIScanPath) aoi_scan_path_analyzers: dict = field(default_factory=dict) - log: bool = field(default=False) + loggers: dict = field(default=dict) draw_parameters: dict = field(default_factory=DEFAULT_ARLAYER_DRAW_PARAMETERS) def __post_init__(self): @@ -139,16 +139,6 @@ class ArLayer(DataFeatures.SharedObject): self.aoi_scene = AOI3DScene.AOI3DScene(self.aoi_scene) - # Prepare logging if needed - self.__ts_logs = {} - - if self.log: - - # Create timestamped buffers to log each aoi scan path analysis - for aoi_scan_path_analyzer_module_path in self.aoi_scan_path_analyzers.keys(): - - self.__ts_logs[aoi_scan_path_analyzer_module_path] = DataFeatures.TimeStampedBuffer() - @classmethod def from_dict(self, layer_data: dict, working_directory: str = None) -> ArLayerType: """Load attributes from dictionary. @@ -294,14 +284,28 @@ class ArLayer(DataFeatures.SharedObject): pass - # Load log status + # Load loggers + new_loggers = {} + try: - new_layer_log = layer_data.pop('log') + new_loggers_value = layer_data.pop('loggers') + + for logger_module_path, logger_parameters in new_loggers_value.items(): + + # Prepend argaze.DataLog path when a single name is provided + if len(logger_module_path.split('.')) == 1: + logger_module_path = f'argaze.DataLog.{logger_module_path}' + + logger_module = importlib.import_module(logger_module_path) + + logger = logger_module.TimeStampedDataLogger(**logger_parameters) + + new_loggers[logger_module_path] = logger except KeyError: - new_layer_log = False + pass # Load image parameters try: @@ -318,7 +322,7 @@ class ArLayer(DataFeatures.SharedObject): new_aoi_matcher, \ new_aoi_scan_path, \ new_aoi_scan_path_analyzers, \ - new_layer_log, \ + new_loggers, \ new_layer_draw_parameters \ ) @@ -350,14 +354,6 @@ class ArLayer(DataFeatures.SharedObject): self.__parent = parent - @property - def logs(self): - """ - Get stored logs - """ - - return self.__ts_logs - def look(self, timestamp: int|float, gaze_movement: GazeFeatures.GazePosition = GazeFeatures.UnvalidGazePosition()) -> dict: """ Project timestamped gaze movement into layer. @@ -439,11 +435,6 @@ class ArLayer(DataFeatures.SharedObject): # Store analysis aoi_scan_path_analysis[aoi_scan_path_analyzer_module_path] = aoi_scan_path_analyzer.analysis - # Log analysis - if self.log: - - self.__ts_logs[aoi_scan_path_analyzer_module_path][timestamp] = aoi_scan_path_analyzer.analysis - elif GazeFeatures.is_saccade(gaze_movement): # Append saccade to aoi scan path @@ -461,12 +452,20 @@ class ArLayer(DataFeatures.SharedObject): # Assess total execution time in ms execution_times['total'] = (time.perf_counter() - look_start) * 1e3 + + # Edit look data + look_data = looked_aoi_name, aoi_scan_path_analysis, execution_times, exception + + # Log look data + for logger_module_path, logger in self.loggers.items(): + + logger.emit(timestamp, look_data) # Unlock layer exploitation self.release() # Return look data - return looked_aoi, aoi_scan_path_analysis, execution_times, exception + return look_data def draw(self, image: numpy.array, draw_aoi_scene: dict = None, draw_aoi_matching: dict = None): """ @@ -538,7 +537,7 @@ class ArFrame(DataFeatures.SharedObject): heatmap: heatmap object background: picture to draw behind layers: dictionary of AOI layers - log: enable scan path analysis logging + loggers: dictionary of timestamped data loggers image_parameters: default parameters passed to image method """ @@ -552,7 +551,7 @@ class ArFrame(DataFeatures.SharedObject): heatmap: AOIFeatures.Heatmap = field(default_factory=AOIFeatures.Heatmap) background: numpy.array = field(default_factory=lambda : numpy.array([])) layers: dict = field(default_factory=dict) - log: bool = field(default=False) + loggers: dict = field(default=dict) image_parameters: dict = field(default_factory=DEFAULT_ARFRAME_IMAGE_PARAMETERS) def __post_init__(self): @@ -571,16 +570,6 @@ class ArFrame(DataFeatures.SharedObject): # Init current gaze position self.__gaze_position = GazeFeatures.UnvalidGazePosition() - # Prepare logging if needed - self.__ts_logs = {} - - if self.log: - - # Create timestamped buffers to log each aoi scan path analysis - for scan_path_analyzer_module_path in self.scan_path_analyzers.keys(): - - self.__ts_logs[scan_path_analyzer_module_path] = DataFeatures.TimeStampedBuffer() - @classmethod def from_dict(self, frame_data: dict, working_directory: str = None) -> ArFrameType: """Load attributes from dictionary. @@ -777,6 +766,29 @@ class ArFrame(DataFeatures.SharedObject): new_frame_log = False + # Load loggers + new_loggers = {} + + try: + + new_loggers_value = frame_data.pop('loggers') + + for logger_module_path, logger_parameters in new_loggers_value.items(): + + # Prepend argaze.DataLog path when a single name is provided + if len(logger_module_path.split('.')) == 1: + logger_module_path = f'argaze.DataLog.{logger_module_path}' + + logger_module = importlib.import_module(logger_module_path) + + logger = logger_module.TimeStampedDataLogger(**logger_parameters) + + new_loggers[logger_module_path] = logger + + except KeyError: + + pass + # Load image parameters try: @@ -797,7 +809,7 @@ class ArFrame(DataFeatures.SharedObject): new_heatmap, \ new_frame_background, \ new_layers, \ - new_frame_log, + new_loggers, new_frame_image_parameters \ ) @@ -829,14 +841,6 @@ class ArFrame(DataFeatures.SharedObject): self.__parent = parent - @property - def logs(self): - """ - Get stored logs - """ - - return self.__ts_logs - def look(self, timestamp: int|float, gaze_position: GazeFeatures.GazePosition = GazeFeatures.UnvalidGazePosition()) -> Tuple[GazeFeatures.GazePosition, GazeFeatures.GazeMovement, dict, dict, dict, Exception]: """ Project gaze position into frame. @@ -869,20 +873,19 @@ class ArFrame(DataFeatures.SharedObject): # Init scan path analysis report scan_step_analysis = {} - # Init layer analysis report - layer_analysis = {} - # Assess pipeline execution times execution_times = { 'gaze_movement_identifier': None, 'scan_step_analyzers':{}, - 'heatmap': None, - 'layers': {} + 'heatmap': None } # Catch any error exception = None + # Init layers look data report + layers_look_data = {} + try: # Apply gaze position calibration @@ -941,11 +944,6 @@ class ArFrame(DataFeatures.SharedObject): # Store analysis scan_step_analysis[scan_path_analyzer_module_path] = scan_path_analyzer.analysis - # Log analysis - if self.log: - - self.__ts_logs[scan_path_analyzer_module_path][timestamp] = scan_path_analyzer.analysis - # No valid finished gaze movement: optionnaly stop in progress identification filtering elif self.gaze_movement_identifier is not None and not self.filter_in_progress_identification: @@ -970,15 +968,7 @@ class ArFrame(DataFeatures.SharedObject): # Note: don't filter valid/unvalid finished/unfished gaze movement to allow layers to reset internally for layer_name, layer in self.layers.items(): - looked_aoi, aoi_scan_path_analysis, layer_execution_times, layer_exception = layer.look(timestamp, identified_gaze_movement) - - layer_analysis[layer_name] = aoi_scan_path_analysis - - execution_times['layers'][layer_name] = layer_execution_times - - if layer_exception: - - raise(layer_exception) + layers_look_data[layer_name] = layer.look(timestamp, identified_gaze_movement) except Exception as e: @@ -987,17 +977,25 @@ class ArFrame(DataFeatures.SharedObject): self.__gaze_position = GazeFeatures.UnvalidGazePosition() identified_gaze_movement = GazeFeatures.UnvalidGazeMovement() scan_step_analysis = {} - layer_analysis = {} exception = e + layers_look_data = {} # Assess total execution time in ms execution_times['total'] = (time.perf_counter() - look_start) * 1e3 + # Edit look data + look_data = self.__gaze_position, identified_gaze_movement, scan_step_analysis, execution_times, exception, layers_look_data + + # Log look data + for logger_module_path, logger in self.loggers.items(): + + logger.emit(timestamp, look_data) + # Unlock frame exploitation self.release() # Return look data - return self.__gaze_position, identified_gaze_movement, scan_step_analysis, layer_analysis, execution_times, exception + return look_data def __image(self, background_weight: float = None, heatmap_weight: float = None, draw_gaze_position_calibrator: dict = None, draw_scan_path: dict = None, draw_layers: dict = None, draw_gaze_positions: dict = None, draw_fixations: dict = None, draw_saccades: dict = None) -> numpy.array: """ diff --git a/src/argaze/DataFeatures.py b/src/argaze/DataFeatures.py index 6e058e8..a52e639 100644 --- a/src/argaze/DataFeatures.py +++ b/src/argaze/DataFeatures.py @@ -171,6 +171,14 @@ class SharedObject(): self._token = token self._lock.release() +class TimeStampedDataLogger(): + """Abstract class to define what should provide a timestamped data logger.""" + + def emit(self, timestamp: TimeStampType, data: any): + """Emit timestamped data to a specific log destination.""" + + raise NotImplementedError('emit() method not implemented') + class TimeStampedBuffer(collections.OrderedDict): """Ordered dictionary to handle timestamped data. ``` diff --git a/src/argaze/DataLog/File.py b/src/argaze/DataLog/File.py new file mode 100644 index 0000000..9253f66 --- /dev/null +++ b/src/argaze/DataLog/File.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +"""Module for file logging. +""" + +__author__ = "Théo de la Hogue" +__credits__ = [] +__copyright__ = "Copyright 2023, Ecole Nationale de l'Aviation Civile (ENAC)" +__license__ = "BSD" + +from typing import TypeVar, Tuple +from dataclasses import dataclass, field +import os, pathlib + +from argaze import DataFeatures + +@dataclass +class TimeStampedDataLogger(DataFeatures.TimeStampedDataLogger): + """Implementation of file logger.""" + + path: str = field(default=None) + """File path where to write data.""" + + separator: chr = field(default=' ') + """Char used to separate timestamp from data""" + + def __post_init__(self): + """Check that folder structure exist and create file.""" + + self.path = pathlib.Path(self.path) + + if not os.path.exists(self.path.parent.absolute()): + os.makedirs(self.path.parent.absolute()) + + # Open file + self._file = open(self.path, 'w', encoding='utf-8', buffering=1) + + def __del__(self): + """Close file.""" + + self._file.close() + + def emit(self, timestamp: DataFeatures.TimeStampType, data: any): + """Write timestamp and data separated by separator char as a new line into file.""" + + # Write into file + print(f'{timestamp}{self.separator}{data}', file=self._file, flush=True) \ No newline at end of file diff --git a/src/argaze/DataLog/__init__.py b/src/argaze/DataLog/__init__.py new file mode 100644 index 0000000..3f5a36f --- /dev/null +++ b/src/argaze/DataLog/__init__.py @@ -0,0 +1,4 @@ +""" +Various data logger handler. +""" +__all__ = ['File'] \ No newline at end of file diff --git a/src/argaze/utils/demo_data/demo_gaze_analysis_setup.json b/src/argaze/utils/demo_data/demo_gaze_analysis_setup.json index f921662..5ae7cf3 100644 --- a/src/argaze/utils/demo_data/demo_gaze_analysis_setup.json +++ b/src/argaze/utils/demo_data/demo_gaze_analysis_setup.json @@ -46,9 +46,21 @@ "n_max": 3 }, "Entropy":{} + }, + "loggers": { + "File" : { + "path": "_export/logs/layer_data.txt", + "separator": "," + } } } }, + "loggers": { + "File" : { + "path": "_export/logs/frame_data.txt", + "separator": "," + } + }, "image_parameters": { "background_weight": 1, "heatmap_weight": 0.5, -- cgit v1.1