From 8539f4fddff5ff212b49579130b5c0a8a639b139 Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Mon, 11 Apr 2022 12:01:12 +0200 Subject: Adding video/data synchronisation feature --- src/argaze/TobiiGlassesPro2/TobiiData.py | 3 ++- src/argaze/TobiiGlassesPro2/TobiiEntities.py | 6 ++--- src/argaze/TobiiGlassesPro2/TobiiVideo.py | 39 +++++++++++++++++++++++----- src/argaze/utils/replay_tobii_session.py | 27 ++++++++++++++----- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/argaze/TobiiGlassesPro2/TobiiData.py b/src/argaze/TobiiGlassesPro2/TobiiData.py index b588ded..0a68ff9 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiData.py +++ b/src/argaze/TobiiGlassesPro2/TobiiData.py @@ -2,6 +2,7 @@ import threading import uuid +import gzip import json import time import queue @@ -9,7 +10,7 @@ import queue from argaze import DataStructures from argaze.TobiiGlassesPro2 import TobiiNetworkInterface -class TobiiSegmentData(DataStructures.DictObject): +class TobiiDataSegment(DataStructures.DictObject): """Handle Tobii Glasses Pro 2 segment data file.""" def __init__(self, segment_data_path): diff --git a/src/argaze/TobiiGlassesPro2/TobiiEntities.py b/src/argaze/TobiiGlassesPro2/TobiiEntities.py index 9dc41bc..c5a7055 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiEntities.py +++ b/src/argaze/TobiiGlassesPro2/TobiiEntities.py @@ -2,10 +2,10 @@ import datetime import json -import gzip import os from argaze import DataStructures +from argaze.TobiiGlassesPro2 import TobiiData, TobiiVideo import av import cv2 as cv @@ -65,10 +65,10 @@ class TobiiSegment: return self.__calibrated def load_data(self): - return TobiiSegmentData(os.path.join(self.__segment_path, TOBII_SEGMENT_DATA_FILENAME)) + return TobiiData.TobiiDataSegment(os.path.join(self.__segment_path, TOBII_SEGMENT_DATA_FILENAME)) def load_video(self): - return TobiiSegmentVideo(os.path.join(self.__segment_path, TOBII_SEGMENT_VIDEO_FILENAME)) + return TobiiVideo.TobiiVideoSegment(os.path.join(self.__segment_path, TOBII_SEGMENT_VIDEO_FILENAME)) class TobiiRecording: """Handle Tobii Glasses Pro 2 recording info and segments.""" diff --git a/src/argaze/TobiiGlassesPro2/TobiiVideo.py b/src/argaze/TobiiGlassesPro2/TobiiVideo.py index 57d64d0..daa562e 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiVideo.py +++ b/src/argaze/TobiiGlassesPro2/TobiiVideo.py @@ -8,15 +8,16 @@ import copy from argaze import DataStructures from argaze.TobiiGlassesPro2 import TobiiNetworkInterface +import cv2 as cv import av import numpy class TobiiVideoFrame(DataStructures.DictObject): """Define tobii video frame""" - def __init__(self, matrix, width, height, pts): + def __init__(self, matrix, width, height): - super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts}) + super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height}) class TobiiVideoSegment(): """Handle Tobii Glasses Pro 2 segment video file.""" @@ -30,6 +31,8 @@ class TobiiVideoSegment(): self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH)) self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT)) + + self.__vts_data_buffer = None def get_path(self): return self.__segment_video_path @@ -46,7 +49,17 @@ class TobiiVideoSegment(): def get_height(self): return self.__height - def frames(self): + def frames(self, vts_data_buffer = None): + """Access to frame iterator and optionnaly setup vide / data timestamp synchronisation through vts data buffer.""" + + self.__vts_data_buffer = vts_data_buffer + + # Enable video / data timestamp synchronisation + if self.__vts_data_buffer != None: + + self.__vts_ts, self.__vts = self.__vts_data_buffer.pop_first() + self.__vts_offset = (self.__vts_ts - self.__vts.vts) + return self.__iter__() def __iter__(self): @@ -60,8 +73,22 @@ class TobiiVideoSegment(): frame = self.__container.decode(self.__stream).__next__() + video_ts = int(frame.time * 1000000) + + # If video / data synchronisation is active + if self.__vts_data_buffer != None: + + if video_ts > self.__vts.vts: + + if len(self.__vts_data_buffer) > 0: + + self.__vts_ts, self.__vts = self.__vts_data_buffer.pop_first() + self.__vts_offset = (self.__vts_ts - self.__vts.vts) + + video_ts += self.__vts_offset + # return micro second timestamp and frame data - return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) + return video_ts, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height) class TobiiVideoStream(threading.Thread): """Capture Tobii Glasses Pro 2 video camera stream.""" @@ -131,7 +158,7 @@ class TobiiVideoStream(threading.Thread): self.__read_lock.acquire() # store frame time, matrix, width, height and pts into a tuple - self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) + self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height) # unlock frame access self.__read_lock.release() @@ -151,4 +178,4 @@ class TobiiVideoStream(threading.Thread): # unlock frame access self.__read_lock.release() - return frame_tuple[0] * 1000000, TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3], frame_tuple[4]) + return int(frame_tuple[0] * 1000000), TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3]) diff --git a/src/argaze/utils/replay_tobii_session.py b/src/argaze/utils/replay_tobii_session.py index b911d09..87f043d 100644 --- a/src/argaze/utils/replay_tobii_session.py +++ b/src/argaze/utils/replay_tobii_session.py @@ -35,25 +35,38 @@ def main(): # Access to timestamped gaze position data buffer tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp - print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded') - # video and data replay loop + # Video and data replay loop try: - for frame_ts, frame in tobii_segment_video.frames(): + # Iterate on video frames activating video / data synchronisation through vts data buffer + for video_ts, video_frame in tobii_segment_video.frames(tobii_segment_data.vts): + + try: + + # Get closest gaze position before video timestamp and remove all gaze positions before + closest_gaze_ts, closest_gaze_position = tobii_ts_gaze_positions.pop_first_until(video_ts) + + # Draw video synchronized gaze pointer + pointer = (int(closest_gaze_position.gp[0] * video_frame.width), int(closest_gaze_position.gp[1] * video_frame.height)) + cv.circle(video_frame.matrix, pointer, 4, (0, 255, 255), -1) + + # When expected values can't be found + except (KeyError, AttributeError, ValueError): + pass - # close window using 'Esc' key + # Close window using 'Esc' key if cv.waitKey(1) == 27: break - cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix) + cv.imshow(f'Segment {tobii_segment.get_id()} video', video_frame.matrix) - # exit on 'ctrl+C' interruption + # Exit on 'ctrl+C' interruption except KeyboardInterrupt: pass - # stop frame display + # Stop frame display cv.destroyAllWindows() if __name__ == '__main__': -- cgit v1.1