From 56c9e7893eebc53010e59740cfbf67808d9db247 Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Wed, 26 Jul 2023 17:27:01 +0200 Subject: Improving looked aoi managment. --- src/argaze/ArFeatures.py | 123 +++++++++++++++++++++++--------- src/argaze/AreaOfInterest/AOI2DScene.py | 18 ++--- 2 files changed, 97 insertions(+), 44 deletions(-) diff --git a/src/argaze/ArFeatures.py b/src/argaze/ArFeatures.py index 020c194..349ad48 100644 --- a/src/argaze/ArFeatures.py +++ b/src/argaze/ArFeatures.py @@ -79,6 +79,7 @@ class ArFrame(): aoi_2d_scene: AOI2DScene.AOI2DScene = field(default_factory=AOI2DScene.AOI2DScene) background: numpy.array = field(default_factory=numpy.array) gaze_movement_identifier: GazeFeatures.GazeMovementIdentifier = field(default_factory=GazeFeatures.GazeMovementIdentifier) + looked_aoi_covering_threshold: int = field(default=0) scan_path: GazeFeatures.ScanPath = field(default_factory=GazeFeatures.ScanPath) scan_path_analyzers: dict = field(default_factory=dict) aoi_scan_path: GazeFeatures.AOIScanPath = field(default_factory=GazeFeatures.AOIScanPath) @@ -93,6 +94,9 @@ class ArFrame(): # Init current gaze position self.__gaze_position = GazeFeatures.UnvalidGazePosition() + # Init looked aoi data + self.__init_looked_aoi_data() + # Init heatmap if required if self.heatmap: @@ -167,6 +171,15 @@ class ArFrame(): finished_gaze_movement_identifier = None + # Looked aoi validity threshold + try: + + looked_aoi_covering_threshold = frame_data.pop('looked_aoi_covering_threshold') + + except KeyError: + + looked_aoi_covering_threshold = 0 + # Load scan path try: @@ -305,6 +318,7 @@ class ArFrame(): new_aoi_2d_scene, \ new_frame_background, \ finished_gaze_movement_identifier, \ + looked_aoi_covering_threshold, \ new_scan_path, \ new_scan_path_analyzers, \ new_aoi_scan_path, \ @@ -361,16 +375,75 @@ class ArFrame(): return image - def look(self, timestamp: int|float, inner_gaze_position: GazeFeatures.GazePosition) -> Tuple[GazeFeatures.GazeMovement, str, dict, dict, dict]: + @property + def looked_aoi(self) -> str: + """Get most likely looked aoi name for current fixation (e.g. the aoi with the highest covering mean value)""" + + return self.__looked_aoi + + @property + def looked_aoi_covering_mean(self) -> float: + """Get looked aoi covering mean for current fixation. + It represents the ratio of fixation deviation circle surface that used to cover the looked aoi.""" + + return self.__looked_aoi_covering_mean + + @property + def looked_aoi_covering(self) -> dict: + """Get all looked aois covering for current fixation.""" + + return self.__looked_aoi_covering + + def __init_looked_aoi_data(self): + """Init looked aoi data.""" + + self.__looked_aoi = None + self.__looked_aoi_covering_mean = 0 + self.__looked_aoi_covering = {} + + def __update_looked_aoi_data(self, fixation): + """Update looked aoi data.""" + + max_covering = 0. + most_likely_looked_aoi = None + + for name, aoi in self.aoi_2d_scene.items(): + + _, _, circle_ratio = aoi.circle_intersection(fixation.focus, fixation.deviation_max) + + if name != self.name and circle_ratio > 0: + + # Sum circle ratio to update aoi covering + try: + + self.__looked_aoi_covering[name] += circle_ratio + + except KeyError: + + self.__looked_aoi_covering[name] = circle_ratio + + # Update most likely aoi + if self.__looked_aoi_covering[name] > max_covering: + + most_likely_looked_aoi = name + max_covering = self.__looked_aoi_covering[name] + + # Update looked aoi + self.__looked_aoi = most_likely_looked_aoi + + # Update looked aoi covering mean + self.__looked_aoi_covering_mean = int(100 * max_covering / (len(fixation.positions) - 2)) / 100 + + def look(self, timestamp: int|float, inner_gaze_position: GazeFeatures.GazePosition) -> Tuple[GazeFeatures.GazeMovement, dict, dict, dict]: """ GazeFeatures.AOIScanStepError Returns: fixation: identified fixation (if gaze_movement_identifier is instanciated) - look at: when identified fixation looks at scan_step: new scan step (if scan_path is instanciated) aoi_scan_step: new scan step (if aoi_scan_path is instanciated) + exception: error catched during gaze position processing """ # Lock frame exploitation @@ -382,9 +455,6 @@ class ArFrame(): # No fixation is identified by default fixation = GazeFeatures.UnvalidGazeMovement() - # No aoi is looked by default - look_at = None - # Init scan path analysis report scan_step_analysis = {} aoi_scan_step_analysis = {} @@ -409,17 +479,7 @@ class ArFrame(): fixation = finished_gaze_movement # Does the fixation match an aoi? - for name, aoi in self.aoi_2d_scene.items(): - - _, _, circle_ratio = aoi.circle_intersection(finished_gaze_movement.focus, finished_gaze_movement.deviation_max) - - if circle_ratio > 0.25: - - if name != self.name: - - # Update current look at - look_at = name - break + self.__update_looked_aoi_data(fixation) # Append fixation to scan path if self.scan_path != None: @@ -427,9 +487,9 @@ class ArFrame(): self.scan_path.append_fixation(timestamp, finished_gaze_movement) # Append fixation to aoi scan path - if self.aoi_scan_path != None and look_at != None: + if self.aoi_scan_path != None and self.looked_aoi != None and self.looked_aoi_covering_mean > self.looked_aoi_covering_threshold: - aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, finished_gaze_movement, look_at) + aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, finished_gaze_movement, self.looked_aoi) # Analyze aoi scan path if aoi_scan_step and len(self.aoi_scan_path) > 1: @@ -442,8 +502,8 @@ class ArFrame(): elif GazeFeatures.is_saccade(finished_gaze_movement): - # Update current look at - look_at = None + # Reset looked aoi + self.__init_looked_aoi_data() # Append saccade to scan path if self.scan_path != None: @@ -475,17 +535,7 @@ class ArFrame(): fixation = current_fixation # Does the fixation match an aoi? - for name, aoi in self.aoi_2d_scene.items(): - - _, _, circle_ratio = aoi.circle_intersection(current_fixation.focus, current_fixation.deviation_max) - - if circle_ratio > 0.25: - - if name != self.name: - - # Update current look at - look_at = name - break + self.__update_looked_aoi_data(fixation) # Update heatmap if self.heatmap: @@ -494,8 +544,9 @@ class ArFrame(): except Exception as e: + print(e) + fixation = GazeFeatures.UnvalidGazeMovement() - look_at = None scan_step_analysis = {} aoi_scan_step_analysis = {} exception = e @@ -504,7 +555,7 @@ class ArFrame(): self.__look_lock.release() # Return look data - return fixation, look_at, scan_step_analysis, aoi_scan_step_analysis, exception + return fixation, scan_step_analysis, aoi_scan_step_analysis, exception def draw(self, image:numpy.array, aoi_color=(0, 0, 0)) -> Exception: """ @@ -538,8 +589,10 @@ class ArFrame(): current_fixation.draw(image, color=(0, 255, 255)) current_fixation.draw_positions(image) - # Draw looked AOI - self.aoi_2d_scene.draw_circlecast(image, current_fixation.focus, current_fixation.deviation_max, base_color=(0, 0, 0), matching_color=(255, 255, 255)) + # Draw looked aoi + if self.looked_aoi_covering_mean > self.looked_aoi_covering_threshold: + + self.aoi_2d_scene.draw_circlecast(image, current_fixation.focus, current_fixation.deviation_max, matching_aoi = [self.__looked_aoi], base_color=(0, 0, 0), matching_color=(255, 255, 255)) current_saccade = self.gaze_movement_identifier.current_saccade diff --git a/src/argaze/AreaOfInterest/AOI2DScene.py b/src/argaze/AreaOfInterest/AOI2DScene.py index 3cf0a89..694e304 100644 --- a/src/argaze/AreaOfInterest/AOI2DScene.py +++ b/src/argaze/AreaOfInterest/AOI2DScene.py @@ -84,31 +84,31 @@ class AOI2DScene(AOIFeatures.AOIScene): yield name, aoi, matching_region, aoi_ratio, circle_ratio - def draw_circlecast(self, image: numpy.array, center:tuple, radius:float, exclude=[], base_color=(0, 0, 255), matching_color=(0, 255, 0)): + def draw_circlecast(self, image: numpy.array, center:tuple, radius:float, matching_aoi = [], exclude=[], base_color=(0, 0, 255), matching_color=(0, 255, 0)): """Draw AOIs with their matching status and matching region.""" for name, aoi, matching_region, aoi_ratio, circle_ratio in self.circlecast(center, radius): if name in exclude: continue + + color = base_color # Draw matching region if aoi_ratio > 0: - matching_region.draw(image, base_color, 4) - - # TODO : Externalise this criteria - matching = aoi_ratio > 0.25 or circle_ratio > 0.5 + matching_region.draw(image, color, 4) - color = matching_color if matching else base_color + # Is aoi part of matching aoi? + if name in matching_aoi: - if matching: + color = matching_color top_left_corner_pixel = numpy.rint(aoi.clockwise()[0]).astype(int) cv2.putText(image, name, top_left_corner_pixel, cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv2.LINE_AA) # Draw matching region matching_region.draw(image, matching_color, 4) - + # Draw form aoi.draw(image, color) @@ -140,4 +140,4 @@ class AOI2DScene(AOIFeatures.AOIScene): aoi2D_scene[name] = numpy.matmul(aoi2D - Src_origin, M.T) - return aoi2D_scene \ No newline at end of file + return aoi2D_scene -- cgit v1.1