diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/argaze/ArFeatures.py | 171 | ||||
| -rw-r--r-- | src/argaze/GazeAnalysis/DeviationCircleCoverage.py | 100 | ||||
| -rw-r--r-- | src/argaze/GazeFeatures.py | 15 | ||||
| -rw-r--r-- | src/argaze/utils/demo_environment/demo_ar_features_setup.json | 4 | ||||
| -rw-r--r-- | src/argaze/utils/demo_environment/demo_gaze_features_setup.json | 5 |
5 files changed, 173 insertions, 122 deletions
diff --git a/src/argaze/ArFeatures.py b/src/argaze/ArFeatures.py index 5fa5cde..4811a0b 100644 --- a/src/argaze/ArFeatures.py +++ b/src/argaze/ArFeatures.py @@ -74,7 +74,7 @@ class ArLayer(): name: name of the layer aoi_color: color to used in draw method aoi_scene: AOI scene description - looked_aoi_covering_threshold: + aoi_matcher: AOI matcher object aoi_scan_path: AOI scan path object aoi_scan_path_analyzers: dictionary of AOI scan path analyzers """ @@ -82,7 +82,7 @@ class ArLayer(): name: str aoi_color: tuple = field(default=(0, 0, 0)) aoi_scene: AOIFeatures.AOIScene = field(default_factory=AOIFeatures.AOIScene) - looked_aoi_covering_threshold: int = field(default=0) + aoi_matcher: GazeFeatures.AOIMatcher = field(default_factory=GazeFeatures.AOIMatcher) aoi_scan_path: GazeFeatures.AOIScanPath = field(default_factory=GazeFeatures.AOIScanPath) aoi_scan_path_analyzers: dict = field(default_factory=dict) @@ -94,9 +94,6 @@ class ArLayer(): # Init current gaze movement self.__gaze_movement = GazeFeatures.UnvalidGazeMovement() - # Init looked aoi data - self.__init_looked_aoi_data() - # Init lock to share looking data with multiples threads self.__look_lock = threading.Lock() @@ -176,14 +173,23 @@ class ArLayer(): # Add AOI 2D Scene by default new_aoi_scene = AOI2DScene.AOI2DScene() - # Looked aoi validity threshold + # Load aoi matcher try: - looked_aoi_covering_threshold = layer_data.pop('looked_aoi_covering_threshold') + aoi_matcher_value = layer_data.pop('aoi_matcher') + + aoi_matcher_module_path, aoi_matcher_parameters = aoi_matcher_value.popitem() + + # Prepend argaze.GazeAnalysis path when a single name is provided + if len(aoi_matcher_module_path.split('.')) == 1: + aoi_matcher_module_path = f'argaze.GazeAnalysis.{aoi_matcher_module_path}' + + aoi_matcher_module = importlib.import_module(aoi_matcher_module_path) + new_aoi_matcher = aoi_matcher_module.AOIMatcher(**aoi_matcher_parameters) except KeyError: - looked_aoi_covering_threshold = 0 + new_aoi_matcher = None # Edit expected AOI list by removing AOI with name equals to layer name expected_aois = list(new_aoi_scene.keys()) @@ -260,7 +266,7 @@ class ArLayer(): return ArLayer(new_layer_name, \ new_aoi_color, \ new_aoi_scene, \ - looked_aoi_covering_threshold, \ + new_aoi_matcher, \ new_aoi_scan_path, \ new_aoi_scan_path_analyzers \ ) @@ -293,70 +299,6 @@ class ArLayer(): self.__parent = parent - @property - def looked_aoi(self) -> str: - """Get most likely looked aoi name for current fixation (e.g. the aoi with the highest covering mean value)""" - - return self.__looked_aoi - - @property - def looked_aoi_covering_mean(self) -> float: - """Get looked aoi covering mean for current fixation. - It represents the ratio of fixation deviation circle surface that used to cover the looked aoi.""" - - return self.__looked_aoi_covering_mean - - @property - def looked_aoi_covering(self) -> dict: - """Get all looked aois covering for current fixation.""" - - return self.__looked_aoi_covering - - def __init_looked_aoi_data(self): - """Init looked aoi data.""" - - self.__look_count = 0 - self.__looked_aoi = None - self.__looked_aoi_covering_mean = 0 - self.__looked_aoi_covering = {} - - def __update_looked_aoi_data(self, fixation) -> str: - """Update looked aoi data.""" - - self.__look_count += 1 - - max_covering = 0. - most_likely_looked_aoi = None - - for name, aoi in self.aoi_scene.items(): - - _, _, circle_ratio = aoi.circle_intersection(fixation.focus, fixation.deviation_max) - - if name != self.name and circle_ratio > 0: - - # Sum circle ratio to update aoi covering - try: - - self.__looked_aoi_covering[name] += circle_ratio - - except KeyError: - - self.__looked_aoi_covering[name] = circle_ratio - - # Update most likely looked aoi - if self.__looked_aoi_covering[name] > max_covering: - - most_likely_looked_aoi = name - max_covering = self.__looked_aoi_covering[name] - - # Update looked aoi - self.__looked_aoi = most_likely_looked_aoi - - # Update looked aoi covering mean - self.__looked_aoi_covering_mean = int(100 * max_covering / self.__look_count) / 100 - - return self.__looked_aoi - def look(self, timestamp: int|float, gaze_movement: GazeFeatures.GazePosition = GazeFeatures.UnvalidGazePosition()) -> dict: """ Project timestamped gaze movement into layer. @@ -387,7 +329,7 @@ class ArLayer(): # Assess pipeline execution times execution_times = { - 'aoi_fixation_matcher': None, + 'aoi_matcher': None, 'aoi_scan_step_analyzers': {} } @@ -396,65 +338,54 @@ class ArLayer(): try: - # Valid and finished gaze movement has been identified - if gaze_movement.valid and gaze_movement.finished: + # Check gaze movement validity + if gaze_movement.valid: - if GazeFeatures.is_fixation(gaze_movement): + if self.aoi_matcher: # Store aoi matching start date matching_start = time.perf_counter() - # Does the finished fixation match an aoi? - looked_aoi = self.__update_looked_aoi_data(gaze_movement) + # Update looked aoi thanks to aoi matcher + # Note: don't filter finished/unfinished fixation/saccade as we don't know how the aoi matcher works internally + looked_aoi = self.aoi_matcher.match(self.aoi_scene, gaze_movement, exclude=[self.name]) # Assess aoi matching time in ms - execution_times['aoi_fixation_matcher'] = (time.perf_counter() - matching_start) * 1e3 - - # Append fixation to aoi scan path - if self.aoi_scan_path != None and self.looked_aoi != None and self.looked_aoi_covering_mean > self.looked_aoi_covering_threshold: - - aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, gaze_movement, self.looked_aoi) + execution_times['aoi_matcher'] = (time.perf_counter() - matching_start) * 1e3 - # Is there a new step? - if aoi_scan_step and len(self.aoi_scan_path) > 1: + # Finished gaze movement has been identified + if gaze_movement.finished: - for aoi_scan_path_analyzer_module_path, aoi_scan_path_analyzer in self.aoi_scan_path_analyzers.items(): + if GazeFeatures.is_fixation(gaze_movement): - # Store aoi scan path analysis start date - aoi_scan_path_analysis_start = time.perf_counter() + # Append fixation to aoi scan path + if self.aoi_scan_path != None and looked_aoi != None: - # Analyze aoi scan path - aoi_scan_path_analyzer.analyze(self.aoi_scan_path) + aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, gaze_movement, looked_aoi) - # Assess aoi scan step analysis time in ms - execution_times['aoi_scan_step_analyzers'][aoi_scan_path_analyzer_module_path] = (time.perf_counter() - aoi_scan_path_analysis_start) * 1e3 + # Is there a new step? + if aoi_scan_step and len(self.aoi_scan_path) > 1: - # Store analysis - aoi_scan_path_analysis[aoi_scan_path_analyzer_module_path] = aoi_scan_path_analyzer.analysis + for aoi_scan_path_analyzer_module_path, aoi_scan_path_analyzer in self.aoi_scan_path_analyzers.items(): - elif GazeFeatures.is_saccade(gaze_movement): + # Store aoi scan path analysis start date + aoi_scan_path_analysis_start = time.perf_counter() - # Reset looked aoi - self.__init_looked_aoi_data() + # Analyze aoi scan path + aoi_scan_path_analyzer.analyze(self.aoi_scan_path) - # Append saccade to aoi scan path - if self.aoi_scan_path != None: + # Assess aoi scan step analysis time in ms + execution_times['aoi_scan_step_analyzers'][aoi_scan_path_analyzer_module_path] = (time.perf_counter() - aoi_scan_path_analysis_start) * 1e3 - self.aoi_scan_path.append_saccade(timestamp, gaze_movement) + # Store analysis + aoi_scan_path_analysis[aoi_scan_path_analyzer_module_path] = aoi_scan_path_analyzer.analysis - # Valid in progress fixation - elif gaze_movement.valid and not gaze_movement.finished: + elif GazeFeatures.is_saccade(gaze_movement): - if GazeFeatures.is_fixation(gaze_movement): + # Append saccade to aoi scan path + if self.aoi_scan_path != None: - # Store aoi matching start date - matching_start = time.perf_counter() - - # Does the finished fixation match an aoi? - looked_aoi = self.__update_looked_aoi_data(gaze_movement) - - # Assess aoi matching time in ms - execution_times['aoi_fixation_matcher'] = (time.perf_counter() - matching_start) * 1e3 + self.aoi_scan_path.append_saccade(timestamp, gaze_movement) except Exception as e: @@ -470,9 +401,9 @@ class ArLayer(): # Sum all execution times total_execution_time = 0 - if execution_times['aoi_fixation_matcher']: + if execution_times['aoi_matcher']: - total_execution_time += execution_times['aoi_fixation_matcher'] + total_execution_time += execution_times['aoi_matcher'] for _, aoi_scan_path_analysis_time in execution_times['aoi_scan_step_analyzers'].items(): @@ -511,9 +442,9 @@ class ArLayer(): self.__gaze_movement.draw_positions(image) # Draw looked aoi - if self.looked_aoi_covering_mean > self.looked_aoi_covering_threshold: + if self.aoi_matcher.looked_aoi != None: - self.aoi_scene.draw_circlecast(image, self.__gaze_movement.focus, self.__gaze_movement.deviation_max, matching_aoi = [self.__looked_aoi], base_color=(0, 0, 0), matching_color=(255, 255, 255)) + self.aoi_scene.draw_circlecast(image, self.__gaze_movement.focus, self.__gaze_movement.deviation_max, matching_aoi = [self.aoi_matcher.looked_aoi], base_color=(0, 0, 0), matching_color=(255, 255, 255)) elif GazeFeatures.is_saccade(self.__gaze_movement): @@ -637,7 +568,7 @@ class ArFrame(): except KeyError: - filter_in_progress_fixation = False + filter_in_progress_fixation = True # Load scan path try: @@ -729,7 +660,7 @@ class ArFrame(): # Create layer new_layer = ArLayer.from_dict(layer_data, working_directory) - # Setup layer + # Project 3D aoi scene layer to get only 2D aoi scene if new_layer.aoi_scene.dimension == 3: new_layer.aoi_scene = new_layer.aoi_scene.orthogonal_projection * new_frame_size @@ -1674,7 +1605,7 @@ class ArEnvironment(): aoi_2d = camera_layer.aoi_scene[frame.name] - # TODO: Add option to use gaze precision circle + # TODO?: Should we prefer to use camera frame AOIMatcher object? if aoi_2d.contains_point(gaze_position.value): inner_x, inner_y = aoi_2d.clockwise().inner_axis(gaze_position.value) diff --git a/src/argaze/GazeAnalysis/DeviationCircleCoverage.py b/src/argaze/GazeAnalysis/DeviationCircleCoverage.py new file mode 100644 index 0000000..8e7acef --- /dev/null +++ b/src/argaze/GazeAnalysis/DeviationCircleCoverage.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +"""Matching algorithm based on fixation's deviation circle coverage over AOI +""" + +__author__ = "Théo de la Hogue" +__credits__ = [] +__copyright__ = "Copyright 2023, Ecole Nationale de l'Aviation Civile (ENAC)" +__license__ = "BSD" + +from typing import TypeVar, Tuple +from dataclasses import dataclass, field +import math + +from argaze import GazeFeatures + +import numpy +import cv2 + +GazeMovementType = TypeVar('GazeMovement', bound="GazeMovement") +# Type definition for type annotation convenience + +@dataclass +class AOIMatcher(GazeFeatures.AOIMatcher): + + coverage_threshold: int|float + """ """ + + def __post_init__(self): + """Init looked aoi data.""" + + self.__look_count = 0 + self.__looked_aoi = None + self.__looked_aoi_coverage_mean = 0 + self.__looked_aoi_coverage = {} + + def match(self, aoi_scene, gaze_movement, exclude=[]) -> str: + """Returns AOI with the maximal fixation's deviation circle coverage if above coverage threshold.""" + + if GazeFeatures.is_fixation(gaze_movement): + + self.__look_count += 1 + + max_coverage = 0. + most_likely_looked_aoi = None + + for name, aoi in aoi_scene.items(): + + _, _, circle_ratio = aoi.circle_intersection(gaze_movement.focus, gaze_movement.deviation_max) + + if name not in exclude and circle_ratio > 0: + + # Sum circle ratio to update aoi coverage + try: + + self.__looked_aoi_coverage[name] += circle_ratio + + except KeyError: + + self.__looked_aoi_coverage[name] = circle_ratio + + # Update most likely looked aoi + if self.__looked_aoi_coverage[name] > max_coverage: + + most_likely_looked_aoi = name + max_coverage = self.__looked_aoi_coverage[name] + + # Update looked aoi + self.__looked_aoi = most_likely_looked_aoi + + # Update looked aoi coverage mean + self.__looked_aoi_coverage_mean = int(100 * max_coverage / self.__look_count) / 100 + + # Return + if self.looked_aoi_coverage_mean > self.coverage_threshold: + + return self.__looked_aoi + + elif GazeFeatures.is_saccade(gaze_movement): + + self.__post_init__() + + @property + def looked_aoi(self) -> str: + """Get most likely looked aoi name for current fixation (e.g. the aoi with the highest coverage mean value)""" + + return self.__looked_aoi + + @property + def looked_aoi_coverage_mean(self) -> float: + """Get looked aoi coverage mean for current fixation. + It represents the ratio of fixation deviation circle surface that used to cover the looked aoi.""" + + return self.__looked_aoi_coverage_mean + + @property + def looked_aoi_coverage(self) -> dict: + """Get all looked aois coverage for current fixation.""" + + return self.__looked_aoi_coverage
\ No newline at end of file diff --git a/src/argaze/GazeFeatures.py b/src/argaze/GazeFeatures.py index bb5f991..33fd562 100644 --- a/src/argaze/GazeFeatures.py +++ b/src/argaze/GazeFeatures.py @@ -15,6 +15,7 @@ import json from inspect import getmembers from argaze import DataStructures +from argaze.AreaOfInterest import AOIFeatures import numpy import pandas @@ -700,6 +701,20 @@ class ScanPathAnalyzer(): raise NotImplementedError('analyze() method not implemented') +class AOIMatcher(): + """Abstract class to define what should provide an AOI matcher algorithm.""" + + def match(self, aoi_scene: AOIFeatures.AOIScene, gaze_movement: GazeMovement, exclude=[]) -> str: + """Which AOI is looked in the scene?""" + + raise NotImplementedError('match() method not implemented') + + @property + def looked_aoi(self) -> str: + """Get most likely looked aoi name.""" + + raise NotImplementedError('looked_aoi getter not implemented') + AOIScanStepType = TypeVar('AOIScanStep', bound="AOIScanStep") # Type definition for type annotation convenience diff --git a/src/argaze/utils/demo_environment/demo_ar_features_setup.json b/src/argaze/utils/demo_environment/demo_ar_features_setup.json index 4e16978..39ace44 100644 --- a/src/argaze/utils/demo_environment/demo_ar_features_setup.json +++ b/src/argaze/utils/demo_environment/demo_ar_features_setup.json @@ -1,5 +1,5 @@ { - "name": "AR Environment Demo", + "name": "ArEnvironment Demo", "aruco_detector": { "dictionary": { "name": "DICT_APRILTAG_16h5" @@ -18,7 +18,7 @@ } }, "scenes": { - "AR Scene Demo" : { + "ArScene Demo" : { "aruco_scene": "aruco_scene.obj", "layers": { "_" : { diff --git a/src/argaze/utils/demo_environment/demo_gaze_features_setup.json b/src/argaze/utils/demo_environment/demo_gaze_features_setup.json index 90ae30f..6e43895 100644 --- a/src/argaze/utils/demo_environment/demo_gaze_features_setup.json +++ b/src/argaze/utils/demo_environment/demo_gaze_features_setup.json @@ -28,6 +28,11 @@ "GrayRectangle": { "aoi_color": [0, 0, 0], "aoi_scene": "aoi_3d_scene.obj", + "aoi_matcher": { + "DeviationCircleCoverage": { + "coverage_threshold": 0.5 + } + }, "aoi_scan_path": { "duration_max": 10000 }, |