From a2f7fb094157f50455828717863271a76d6986bd Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Fri, 6 May 2022 00:32:11 +0200 Subject: Fixing look_at and looked_pixel functions. Fixing AOIScene serialization. --- src/argaze/AreaOfInterest/AOI2DScene.py | 7 +- src/argaze/AreaOfInterest/AOI3DScene.py | 17 ++-- src/argaze/AreaOfInterest/AOIFeatures.py | 66 ++++++++------ src/argaze/DataStructures.py | 4 +- src/argaze/GazeFeatures.py | 2 +- .../export_tobii_segment_aruco_visual_scan.py | 100 ++++++++++++++++----- src/argaze/utils/live_tobii_aruco_aois.py | 2 +- 7 files changed, 133 insertions(+), 65 deletions(-) (limited to 'src') diff --git a/src/argaze/AreaOfInterest/AOI2DScene.py b/src/argaze/AreaOfInterest/AOI2DScene.py index c48210a..a120a12 100644 --- a/src/argaze/AreaOfInterest/AOI2DScene.py +++ b/src/argaze/AreaOfInterest/AOI2DScene.py @@ -23,7 +23,7 @@ class AOI2DScene(AOIFeatures.AOIScene): looked = {} ignored = {} - for name, aoi in self.areas.items(): + for name, aoi in self.items(): if aoi.looked(): @@ -38,14 +38,15 @@ class AOI2DScene(AOIFeatures.AOIScene): def draw(self, frame, gaze_position: GazeFeatures.GazePosition): """Draw AOI polygons on frame.""" - for name, aoi2D in self.areas.items(): + for name, aoi2D in self.items(): looked = aoi2D.looked(gaze_position) color = (0, 255, 0) if looked else (0, 0, 255) if looked: - cv.putText(frame, name, aoi2D[3], cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA) + top_left_corner_pixel = numpy.rint(aoi2D.clockwise()[0]).astype(int) + cv.putText(frame, name, top_left_corner_pixel, cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1, cv.LINE_AA) looked_pixel = aoi2D.looked_pixel(aoi2D.look_at(gaze_position)) cv.circle(frame, looked_pixel, 8, color, 2) diff --git a/src/argaze/AreaOfInterest/AOI3DScene.py b/src/argaze/AreaOfInterest/AOI3DScene.py index 28150c6..658fa64 100644 --- a/src/argaze/AreaOfInterest/AOI3DScene.py +++ b/src/argaze/AreaOfInterest/AOI3DScene.py @@ -10,6 +10,9 @@ from argaze.AreaOfInterest import AOIFeatures, AOI2DScene import numpy import cv2 as cv +# Define a zero distorsion matrix +D0 = numpy.asarray([0.0, 0.0, 0.0, 0.0, 0.0]) + @dataclass class AOI3DScene(AOIFeatures.AOIScene): """Define AOI 3D scene.""" @@ -87,25 +90,23 @@ class AOI3DScene(AOIFeatures.AOIScene): # retreive all aoi3D vertices for name, face in faces.items(): - aoi3D = numpy.array([ vertices[i-1] for i in face ]).astype('float32').view(AOIFeatures.AreaOfInterest) - self.append(name, aoi3D) + aoi3D = numpy.array([ vertices[i-1] for i in face ]).astype(numpy.float32).view(AOIFeatures.AreaOfInterest) + self[name] = aoi3D except IOError: raise IOError(f'File not found: {obj_filepath}') - def project(self, K, D): + def project(self, K, D=D0): """Project 3D scene onto 2D scene according optical parameters. **Returns:** AOI2DScene""" aoi2D_scene = AOI2DScene.AOI2DScene() - for name, aoi3D in self.areas.items(): + for name, aoi3D in self.items(): vertices_2D, J = cv.projectPoints(aoi3D, self.rotation, self.translation, K, D) - vertices_2D = vertices_2D.astype('int').reshape((len(vertices_2D), 2)).tolist() - - aoi2D = numpy.array(vertices_2D).view(AOIFeatures.AreaOfInterest) + aoi2D = vertices_2D.reshape((len(vertices_2D), 2)).astype(numpy.float32).view(AOIFeatures.AreaOfInterest) - aoi2D_scene.append(name, aoi2D) + aoi2D_scene[name] = aoi2D return aoi2D_scene diff --git a/src/argaze/AreaOfInterest/AOIFeatures.py b/src/argaze/AreaOfInterest/AOIFeatures.py index 001affa..4495df5 100644 --- a/src/argaze/AreaOfInterest/AOIFeatures.py +++ b/src/argaze/AreaOfInterest/AOIFeatures.py @@ -40,21 +40,21 @@ class AreaOfInterest(numpy.ndarray): return mpath.Path(self).contains_points([gaze_position])[0] - def look_at(self, gaze_position): - """Get where the area is looked using non orthogonal projection.""" + def look_at(self, gaze_pixel): + """Get where the area is looked using perpespective transformation.""" if self.dimension() != 2: raise RuntimeError(f'Bad area dimension ({self.dimension()})') - clockwise_area = self.clockwise() + Src = self.clockwise() + Src_origin = Src[0] + Src = (Src - Src_origin).reshape((len(Src)), 2) - O = clockwise_area[0] # Origin - G = numpy.array(gaze_position) - O # Gaze point + Dst = numpy.array([[0., 0.], [1., 0.], [1., 1.], [0., 1.]]).astype(numpy.float32) - M = numpy.array([clockwise_area[1] - O, clockwise_area[-1] - O]) # Basis projection matrix M = {U | V} - Mt = numpy.transpose(M) - - Gp = numpy.dot(numpy.dot(numpy.linalg.inv(numpy.dot(Mt, M)), Mt), G)# Projected gaze point + P = cv.getPerspectiveTransform(Src, Dst) + G = gaze_pixel - Src_origin + Gp = numpy.dot(P, numpy.array([G[0], G[1], 1]))[:-1] return numpy.around(Gp, 4).tolist() @@ -64,40 +64,54 @@ class AreaOfInterest(numpy.ndarray): if self.dimension() != 2: raise RuntimeError(f'Bad area dimension ({self.dimension()})') - clockwise_area = self.clockwise() + Src = numpy.array([[0., 0.], [1., 0.], [1., 1.], [0., 1.]]).astype(numpy.float32) - O = clockwise_area[0] # Origin - Gp = numpy.array(look_at) # Projected gaze point - M = numpy.array([clockwise_area[1] - O, clockwise_area[-1] - O]) # Basis projection matrix M = {U | V} - Mt = numpy.transpose(M) + Dst = self.clockwise() + Dst_origin = Dst[0] + Dst = (Dst - Dst_origin).reshape((len(Dst)), 2) - Lp = O + numpy.dot(M, Gp) # Projected gaze pixel + P = cv.getPerspectiveTransform(Src, Dst) + L = look_at + Lp = Dst_origin + numpy.dot(P, numpy.array([L[0], L[1], 1]))[:-1] return numpy.rint(Lp).astype(int).tolist() def draw(self, frame, color): # Draw form - cv.line(frame, self[-1], self[0], color, 1) - for A, B in zip(self, self[1:]): + pixels = numpy.rint(self).astype(int) + cv.line(frame, pixels[-1], pixels[0], color, 1) + for A, B in zip(pixels, pixels[1:]): cv.line(frame, A, B, color, 1) # Draw center - cv.circle(frame, self.center().astype(int), 1, color, -1) + center_pixel = numpy.rint(self.center()).astype(int) + cv.circle(frame, center_pixel, 1, color, -1) @dataclass class AOIScene(): - """Define 2D/3D AOI scene.""" + """Define 2D/3D AOI scene.""" - dimension: int = field(init=False, default=None) - """Dimension of the AOIs in scene.""" + dimension: int = field(init=False, repr=False, default=None) + """Dimension of the AOIs in scene.""" - areas: dict = field(init=False, default_factory=dict) - """All aois in the scene.""" + areas: dict = field(init=False, default_factory=dict) + """All aois in the scene.""" - def append(self, name, aoi: AreaOfInterest): - """Add an aoi to the scene.""" - self.areas[name] = aoi + def __getitem__(self, key): + """Get an aoi from the scene.""" + return numpy.array(self.areas[key]).astype(numpy.float32).view(AreaOfInterest) + + def __setitem__(self, name, aoi: AreaOfInterest): + """Add an aoi to the scene.""" + self.areas[name] = aoi.tolist() + + def items(self): + for name, area in self.areas.items(): + yield name, numpy.array(area).astype(numpy.float32).view(AreaOfInterest) + + def keys(self): + return self.areas.keys() class TimeStampedAOIScenes(DataStructures.TimeStampedBuffer): """Define timestamped buffer to store AOI scenes in time.""" diff --git a/src/argaze/DataStructures.py b/src/argaze/DataStructures.py index abd75a8..8517378 100644 --- a/src/argaze/DataStructures.py +++ b/src/argaze/DataStructures.py @@ -29,7 +29,7 @@ class DictObject(): __type_value = self.__dict__.pop(__type_key) self.__dict__.update({key:value}) self.__dict__[__type_key] = __type_value - + class TimeStampedBuffer(collections.OrderedDict): """Ordered dictionary to handle timestamped data. ``` @@ -52,7 +52,7 @@ class TimeStampedBuffer(collections.OrderedDict): super().__setitem__(key, value) def __str__(self): - return json.dumps(self) + return json.dumps(self, default=vars) def append(self, timestamped_buffer): """Append a timestamped buffer.""" diff --git a/src/argaze/GazeFeatures.py b/src/argaze/GazeFeatures.py index ebe6fe5..32ec571 100644 --- a/src/argaze/GazeFeatures.py +++ b/src/argaze/GazeFeatures.py @@ -303,7 +303,7 @@ class PointerBasedVisualScan(VisualScanGenerator): gaze_position = self.__ts_gaze_positions[ts_current] - for name, aoi in aoi_scene_current.areas.items(): + for name, aoi in aoi_scene_current.items(): looked = aoi.looked(gaze_position) diff --git a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py index 4622b00..4f84943 100644 --- a/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py +++ b/src/argaze/utils/export_tobii_segment_aruco_visual_scan.py @@ -14,6 +14,17 @@ import numpy import cv2 as cv +aoi_color = { + 'Scene_Plan': (127, 127, 127), + 'PFD_Plan': (63, 127, 63), + 'Attitude_Plan': (0, 255, 0), + 'Air_Speed_Plan': (255, 0, 255), + 'Vertical_Speed_Plan': (255, 255, 0), + 'Localiser_Plan': (0, 0, 255), + 'ND_Plan': (127, 63, 63), + 'Marker_Plan': (0, 0, 0) +} + def main(): """ Track any ArUco marker into Tobii Glasses Pro 2 segment video file. @@ -50,13 +61,15 @@ def main(): os.makedirs(os.path.dirname(args.output)) print(f'{os.path.dirname(args.output)} folder created') - visual_scan_filepath = f'{args.output}/visual_scan.csv' - video_filepath = f'{args.output}/fullstream+visu.mp4' + vs_data_filepath = f'{args.output}/visual_scan.csv' + vs_visu_filepath = f'{args.output}/visual_scan.jpg' + vs_video_filepath = f'{args.output}/visual_scan.mp4' else: - visual_scan_filepath = f'{args.segment_path}/visual_scan.csv' - video_filepath = f'{args.segment_path}/fullstream+visu.mp4' + vs_data_filepath = f'{args.segment_path}/visual_scan.csv' + vs_visu_filepath = f'{args.segment_path}/visual_scan.jpg' + vs_video_filepath = f'{args.segment_path}/visual_scan.mp4' # Load a tobii segment tobii_segment = TobiiEntities.TobiiSegment(args.segment_path, int(args.time_range[0] * 1000000), int(args.time_range[1] * 1000000) if args.time_range[1] != None else None) @@ -69,12 +82,16 @@ def main(): tobii_segment_data = tobii_segment.load_data() print(f'Data keys: {tobii_segment_data.keys()}') - # Access to timestamped gaze position data buffer + # Access to timestamped gaze positions data buffer tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded') + # Access to timestamped gaze 3D positions data buffer + #tobii_ts_gaze_3d_positions = tobii_segment_data.gidx_gp3 + #print(f'{len(tobii_ts_gaze_3d_positions)} gaze 3D positions loaded') + # Prepare video exportation at the same format than segment video - output_video = TobiiVideo.TobiiVideoOutput(video_filepath, tobii_segment_video.get_stream()) + output_video = TobiiVideo.TobiiVideoOutput(vs_video_filepath, tobii_segment_video.get_stream()) # Create aruco camera aruco_camera = ArUcoCamera.ArUcoCamera() @@ -86,7 +103,7 @@ def main(): # Create AOIs 3D scene aoi3D_scene = AOI3DScene.AOI3DScene() aoi3D_scene.load(args.aoi_scene) - print(f'AOIs names: {aoi3D_scene.areas.keys()}') + print(f'AOIs names: {aoi3D_scene.keys()}') # Create timestamped buffer to store AOIs scene in time ts_aois_scenes = AOIFeatures.TimeStampedAOIScenes() @@ -94,6 +111,26 @@ def main(): # Create timestamped buffer to store gaze positions in time ts_gaze_positions = GazeFeatures.TimeStampedGazePositions() + # Create a visual scan visualisation frame + visu_width = 1920 + visu_height = 1080 + visu_ratio = visu_height + visu_frame = numpy.full((visu_height, visu_width, 3), 255, dtype=numpy.uint8) + + # Project 3D scene on the reference frame + # TODO : center projection on a reference AOI + # TODO: pass the reference AOI in argument + aoi3D_scene.rotation = numpy.asarray([[-numpy.pi, 0.0, 0.0]]) + aoi3D_scene.translation = numpy.asarray([[25.0, -32.0, 20.0]]) + + # Edit a projection matrix for the reference frame + K0 = numpy.asarray([[visu_ratio, 0.0, visu_width/2], [0.0, visu_ratio, visu_height/2], [0.0, 0.0, 1.0]]) + + aoi2D_visu_scene = aoi3D_scene.project(K0) + + for name, aoi in aoi2D_visu_scene.items(): + aoi.draw(visu_frame, aoi_color[name]) + # Video and data replay loop try: @@ -109,11 +146,11 @@ def main(): closest_gaze_ts, closest_gaze_position = tobii_ts_gaze_positions.pop_first_until(video_ts) # Draw gaze position - gaze_position = (int(closest_gaze_position.gp[0] * video_frame.width), int(closest_gaze_position.gp[1] * video_frame.height)) - cv.circle(video_frame.matrix, gaze_position, 4, (0, 255, 255), -1) + video_gaze_pixel = (int(closest_gaze_position.gp[0] * video_frame.width), int(closest_gaze_position.gp[1] * video_frame.height)) + cv.circle(video_frame.matrix, video_gaze_pixel, 4, (0, 255, 255), -1) # Store gaze position at this time in millisecond - ts_gaze_positions[video_ts/1000] = gaze_position + ts_gaze_positions[round(video_ts/1000)] = video_gaze_pixel # Wait for gaze position except ValueError: @@ -123,7 +160,7 @@ def main(): aruco_tracker.track(video_frame.matrix) aruco_tracker.draw(video_frame.matrix) - # Project 3D scene related to each aruco marker + # Project 3D scene on each video frame and the visualisation frame if aruco_tracker.get_markers_number(): for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()): @@ -137,19 +174,30 @@ def main(): aoi3D_scene.rotation = aruco_tracker.get_marker_rotation(i) aoi3D_scene.translation = aruco_tracker.get_marker_translation(i) - - # Edit Zero distorsion matrix - D0 = numpy.asarray([0.0, 0.0, 0.0, 0.0, 0.0]) # DON'T APPLY CAMERA DISTORSION : it projects points which are far from the frame into it # This hack isn't realistic but as the gaze will mainly focus on centered AOI, where the distorsion is low, it is acceptable. - aoi2D_scene = aoi3D_scene.project(aruco_camera.get_K(), D0) + aoi2D_video_scene = aoi3D_scene.project(aruco_camera.get_K()) - # Draw 2D scene - aoi2D_scene.draw(video_frame.matrix, gaze_position) + # Draw 2D scene on video frame + aoi2D_video_scene.draw(video_frame.matrix, video_gaze_pixel) # Store 2D scene at this time in millisecond - ts_aois_scenes[video_ts/1000] = aoi2D_scene + ts_aois_scenes[round(video_ts/1000)] = aoi2D_video_scene + + # Draw gaze path + for name, aoi in aoi2D_video_scene.items(): + + if not aoi.looked(video_gaze_pixel): + continue + + ref_aoi = name #'Scene_Plan' + + look_at = aoi2D_video_scene[ref_aoi].look_at(video_gaze_pixel) + + visu_gaze_pixel = aoi2D_visu_scene[ref_aoi].looked_pixel(look_at) + + cv.circle(visu_frame, visu_gaze_pixel, 4, aoi_color[ref_aoi], -1) # Close window using 'Esc' key if cv.waitKey(1) == 27: @@ -158,6 +206,9 @@ def main(): # Display video cv.imshow(f'Segment {tobii_segment.get_id()} video', video_frame.matrix) + # Display visual scan frame + cv.imshow(f'Segment {tobii_segment.get_id()} visual scan', visu_frame) + # Write video output_video.write(video_frame.matrix) @@ -174,18 +225,19 @@ def main(): # End output video file output_video.close() - - print(f'\nAOIs video saved into {video_filepath}') + print(f'\nVisual scan video saved into {vs_video_filepath}') # Build visual scan based on a pointer position visual_scan = GazeFeatures.PointerBasedVisualScan(ts_aois_scenes, ts_gaze_positions) - print(f'{len(visual_scan.steps())} visual scan steps found') - # Export visual scan - visual_scan.export_as_csv(visual_scan_filepath) + # Export visual scan data + visual_scan.export_as_csv(vs_data_filepath) + print(f'Visual scan data saved into {vs_data_filepath}') - print(f'Visual scan saved into {visual_scan_filepath}') + # Export visual scan image + cv.imwrite(vs_visu_filepath, visu_frame) + print(f'Visual scan image saved into {vs_visu_filepath}') if __name__ == '__main__': diff --git a/src/argaze/utils/live_tobii_aruco_aois.py b/src/argaze/utils/live_tobii_aruco_aois.py index 4f00dea..bf91eec 100644 --- a/src/argaze/utils/live_tobii_aruco_aois.py +++ b/src/argaze/utils/live_tobii_aruco_aois.py @@ -57,7 +57,7 @@ def main(): # Create AOIs 3D scene aoi3D_scene = AOI3DScene.AOI3DScene() aoi3D_scene.load(args.aoi_scene) - print(f'AOIs names: {aoi3D_scene.areas.keys()}') + print(f'AOIs names: {aoi3D_scene.keys()}') # Start streaming tobii_controller.start_streaming() -- cgit v1.1