diff options
| -rw-r--r-- | src/argaze/ArFeatures.py | 106 | ||||
| -rw-r--r-- | src/argaze/utils/demo_ar_features_run.py | 8 |
2 files changed, 80 insertions, 34 deletions
diff --git a/src/argaze/ArFeatures.py b/src/argaze/ArFeatures.py index 7f1618c..d3ce874 100644 --- a/src/argaze/ArFeatures.py +++ b/src/argaze/ArFeatures.py @@ -13,6 +13,7 @@ import json import os import importlib from inspect import getmembers +import threading from argaze import DataStructures, GazeFeatures from argaze.ArUcoMarkers import * @@ -64,6 +65,9 @@ class ArEnvironment(): # Init AOI scene projections self.__aoi_2d_scenes = {} + # Init a lock to share AOI scene projections between multiple threads + self.__aoi_2d_scenes_lock = threading.Lock() + @classmethod def from_json(self, json_filepath: str) -> ArSceneType: """ @@ -303,21 +307,6 @@ class ArEnvironment(): return output @property - def image(self): - """Get ArUco detection visualisation and scenes projections.""" - - # Draw detected markers - self.aruco_detector.draw_detected_markers(self.__image) - - # Draw each AOI scene - for scene_name, aoi_2d_scene in self.__aoi_2d_scenes.items(): - - # Draw AOI scene projection - aoi_2d_scene.draw(self.__image, color=(255, 255, 255)) - - return self.__image - - @property def screens(self): """Iterate over all environment screens""" @@ -332,34 +321,51 @@ class ArEnvironment(): def detect_and_project(self, image: numpy.array) -> dict: """Detect environment aruco markers from image and project scenes.""" - self.__image = image - # Detect aruco markers - self.aruco_detector.detect_markers(self.__image) + self.aruco_detector.detect_markers(image) # Project each AOI scene - self.__aoi_2d_scenes = {} + new_aoi_2d_scenes = {} for scene_name, scene in self.scenes.items(): # Project scene try: # Try to build AOI scene from detected ArUco marker corners - self.__aoi_2d_scenes[scene_name] = scene.build_aruco_aoi_scene(self.aruco_detector.detected_markers) + new_aoi_2d_scenes[scene_name] = scene.build_aruco_aoi_scene(self.aruco_detector.detected_markers) - except: + except SceneProjectionFailed: # Estimate scene markers poses self.aruco_detector.estimate_markers_pose(scene.aruco_scene.identifiers) - + # Estimate scene pose from detected scene markers tvec, rmat, _, _ = scene.estimate_pose(self.aruco_detector.detected_markers) - + # Project AOI scene into video image according estimated pose - self.__aoi_2d_scenes[scene_name] = scene.project(tvec, rmat) + new_aoi_2d_scenes[scene_name] = scene.project(tvec, rmat) + + # Lock scene projections exploitation + self.__aoi_2d_scenes_lock.acquire() + + # Copy scene projections + self.__aoi_2d_scenes = new_aoi_2d_scenes.copy() + + # Unlock scene projections exploitation + self.__aoi_2d_scenes_lock.release() def look(self, timestamp: int|float, gaze_position: GazeFeatures.GazePosition): - """Project gaze position into environment at particular time.""" + """Project timestamped gaze position into current scene projections.""" + + # Can't use scene projection when it is locked + if self.__aoi_2d_scenes_lock.locked(): + + #TODO: Store ignored timestamped gaze positions for further projections + print('Ignoring ', timestamp, gaze_position) + return + + # Lock scene projections + self.__aoi_2d_scenes_lock.acquire() # For each aoi scene projection for scene_name, scene in self.scenes.items(): @@ -379,13 +385,19 @@ class ArEnvironment(): # QUESTION: How to project gaze precision? inner_gaze_position = GazeFeatures.GazePosition((inner_x, inner_y)) - screen.look(timestamp, inner_gaze_position * screen.size) + gaze_movement, look_at, scan_step, aoi_scan_step = screen.look(timestamp, inner_gaze_position * screen.size) + + # NOT COMPATIBLE without for loop + #yield scene_name, screen_name, screen, gaze_movement, look_at, scan_step, aoi_scan_step # Ignore missing aoi scene projection except KeyError: pass + # Unlock scene projections + self.__aoi_2d_scenes_lock.release() + def to_json(self, json_filepath): """Save environment to .json file.""" @@ -393,6 +405,18 @@ class ArEnvironment(): json.dump(self, file, ensure_ascii=False, indent=4, cls=DataStructures.JsonEncoder) + def draw(self, image: numpy.array): + """Draw ArUco detection visualisation and scenes projections.""" + + # Draw detected markers + self.aruco_detector.draw_detected_markers(image) + + # Draw each AOI scene + for scene_name, aoi_2d_scene in self.__aoi_2d_scenes.items(): + + # Draw AOI scene projection + aoi_2d_scene.draw(image, color=(255, 255, 255)) + class PoseEstimationFailed(Exception): """ Exception raised by ArScene estimate_pose method when the pose can't be estimated due to unconsistencies. @@ -479,7 +503,7 @@ class ArScene(): # Pose estimation fails when no marker is detected if len(detected_markers) == 0: - + raise PoseEstimationFailed('No marker detected') scene_markers, _ = self.aruco_scene.filter_markers(detected_markers) @@ -575,6 +599,11 @@ class ArScene(): built AOI 2D scene """ + # Check aruco aoi is defined + if len(self.aruco_aoi) == 0: + + raise SceneProjectionFailed('No aruco aoi is defined') + # AOI projection fails when no marker is detected if len(detected_markers) == 0: @@ -713,10 +742,21 @@ class ArScreen(): """ GazeFeatures.AOIScanStepError + + Returns: + gaze_movement: identified gaze movement (if gaze_movement_identifier is intanciated) + scan_step: new scan step (if scan_path is intanciated) + aoi_scan_step: new scan step (if aoi_scan_path is intanciated) """ self.__gaze_position = inner_gaze_position + # Prepare return + gaze_movement = None + look_at = self.name + scan_step = None + aoi_scan_step = None + # Identify gaze movement if self.gaze_movement_identifier: @@ -726,7 +766,6 @@ class ArScreen(): if GazeFeatures.is_fixation(gaze_movement): # Does the fixation match an AOI? - look_at = self.name for name, aoi in self.aoi_2d_scene.items(): _, _, circle_ratio = aoi.circle_intersection(gaze_movement.focus, gaze_movement.deviation_max) @@ -746,10 +785,10 @@ class ArScreen(): # Append fixation to aoi scan path if self.aoi_scan_path != None: - self.__aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, gaze_movement, look_at) + aoi_scan_step = self.aoi_scan_path.append_fixation(timestamp, gaze_movement, look_at) # Analyze aoi scan path - if self.__aoi_scan_step and len(self.aoi_scan_path) > 1: + if aoi_scan_step and len(self.aoi_scan_path) > 1: for aoi_scan_path_analyzer_type, aoi_scan_path_analyzer in self.aoi_scan_path_analyzers.items(): @@ -760,10 +799,10 @@ class ArScreen(): # Append saccade to scan path if self.scan_path != None: - self.__scan_step = self.scan_path.append_saccade(timestamp, gaze_movement) + scan_step = self.scan_path.append_saccade(timestamp, gaze_movement) # Analyze aoi scan path - if self.__scan_step and len(self.scan_path) > 1: + if scan_step and len(self.scan_path) > 1: for scan_path_analyzer_type, scan_path_analyzer in self.scan_path_analyzers.items(): @@ -778,3 +817,6 @@ class ArScreen(): if self.heatmap: self.heatmap.update(self.__gaze_position.value, sigma=0.05) + + # Return + return gaze_movement, look_at, scan_step, aoi_scan_step diff --git a/src/argaze/utils/demo_ar_features_run.py b/src/argaze/utils/demo_ar_features_run.py index 39c69e7..0f2c2c2 100644 --- a/src/argaze/utils/demo_ar_features_run.py +++ b/src/argaze/utils/demo_ar_features_run.py @@ -76,9 +76,12 @@ def main(): cv2.putText(video_image, f'Error: {e}', (20, 80), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 255), 1, cv2.LINE_AA) # Draw environment - cv2.imshow(ar_environment.name, ar_environment.image) + ar_environment.draw(video_image) - # Draw each screens + # Display environment + cv2.imshow(ar_environment.name, video_image) + + # Draw and display each screens for scene_name, screen_name, screen in ar_environment.screens: image = screen.background.copy() @@ -95,6 +98,7 @@ def main(): # Draw looked AOI screen.aoi_2d_scene.draw_circlecast(image, screen.current_gaze_movement.focus, screen.current_gaze_movement.deviation_max, base_color=(0, 0, 0), matching_color=(255, 255, 255)) + # Display screen cv2.imshow(f'{scene_name}:{screen_name}', image) # Stop by pressing 'Esc' key |