Adding video/data synchronisation feature

4 files changed, 58 insertions, 17 deletions

diff --git a/src/argaze/TobiiGlassesPro2/TobiiData.py b/src/argaze/TobiiGlassesPro2/TobiiData.py
index b588ded..0a68ff9 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiData.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiData.py
@@ -2,6 +2,7 @@
 
 import threading
 import uuid
+import gzip
 import json
 import time
 import queue
@@ -9,7 +10,7 @@ import queue
 from argaze import DataStructures
 from argaze.TobiiGlassesPro2 import TobiiNetworkInterface
 
-class TobiiSegmentData(DataStructures.DictObject):
+class TobiiDataSegment(DataStructures.DictObject):
     """Handle Tobii Glasses Pro 2 segment data file."""
 
     def __init__(self, segment_data_path):
diff --git a/src/argaze/TobiiGlassesPro2/TobiiEntities.py b/src/argaze/TobiiGlassesPro2/TobiiEntities.py
index 9dc41bc..c5a7055 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiEntities.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiEntities.py
@@ -2,10 +2,10 @@
 
 import datetime
 import json
-import gzip
 import os
 
 from argaze import DataStructures
+from argaze.TobiiGlassesPro2 import TobiiData, TobiiVideo
 
 import av
 import cv2 as cv
@@ -65,10 +65,10 @@ class TobiiSegment:
         return self.__calibrated
 
     def load_data(self):
-        return TobiiSegmentData(os.path.join(self.__segment_path, TOBII_SEGMENT_DATA_FILENAME))
+        return TobiiData.TobiiDataSegment(os.path.join(self.__segment_path, TOBII_SEGMENT_DATA_FILENAME))
 
     def load_video(self):
-        return TobiiSegmentVideo(os.path.join(self.__segment_path, TOBII_SEGMENT_VIDEO_FILENAME))
+        return TobiiVideo.TobiiVideoSegment(os.path.join(self.__segment_path, TOBII_SEGMENT_VIDEO_FILENAME))
 
 class TobiiRecording:
     """Handle Tobii Glasses Pro 2 recording info and segments."""
diff --git a/src/argaze/TobiiGlassesPro2/TobiiVideo.py b/src/argaze/TobiiGlassesPro2/TobiiVideo.py
index 57d64d0..daa562e 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiVideo.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiVideo.py
@@ -8,15 +8,16 @@ import copy
 from argaze import DataStructures
 from argaze.TobiiGlassesPro2 import TobiiNetworkInterface
 
+import cv2 as cv
 import av
 import numpy
 
 class TobiiVideoFrame(DataStructures.DictObject):
     """Define tobii video frame"""
 
-    def __init__(self, matrix, width, height, pts):
+    def __init__(self, matrix, width, height):
 
-        super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts})
+        super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height})
 
 class TobiiVideoSegment():
     """Handle Tobii Glasses Pro 2 segment video file."""
@@ -30,6 +31,8 @@ class TobiiVideoSegment():
 
         self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH))
         self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT))
+
+        self.__vts_data_buffer = None
         
     def get_path(self):
         return self.__segment_video_path
@@ -46,7 +49,17 @@ class TobiiVideoSegment():
     def get_height(self):
         return self.__height
 
-    def frames(self):
+    def frames(self, vts_data_buffer = None):
+        """Access to frame iterator and optionnaly setup vide / data timestamp synchronisation through vts data buffer."""
+
+        self.__vts_data_buffer = vts_data_buffer
+
+        # Enable video / data timestamp synchronisation
+        if self.__vts_data_buffer != None:
+
+            self.__vts_ts, self.__vts = self.__vts_data_buffer.pop_first()
+            self.__vts_offset = (self.__vts_ts - self.__vts.vts)
+
         return self.__iter__()
 
     def __iter__(self):
@@ -60,8 +73,22 @@ class TobiiVideoSegment():
 
         frame = self.__container.decode(self.__stream).__next__()
 
+        video_ts = int(frame.time * 1000000)
+
+        # If video / data synchronisation is active
+        if self.__vts_data_buffer != None:
+
+            if video_ts > self.__vts.vts:
+
+                if len(self.__vts_data_buffer) > 0:
+
+                    self.__vts_ts, self.__vts = self.__vts_data_buffer.pop_first()
+                    self.__vts_offset = (self.__vts_ts - self.__vts.vts)
+
+            video_ts += self.__vts_offset
+
         # return micro second timestamp and frame data
-        return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) 
+        return video_ts, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height)
 
 class TobiiVideoStream(threading.Thread):
     """Capture Tobii Glasses Pro 2 video camera stream."""
@@ -131,7 +158,7 @@ class TobiiVideoStream(threading.Thread):
             self.__read_lock.acquire()
 
             # store frame time, matrix, width, height and pts into a tuple
-            self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts)
+            self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height)
 
             # unlock frame access
             self.__read_lock.release()
@@ -151,4 +178,4 @@ class TobiiVideoStream(threading.Thread):
         # unlock frame access
         self.__read_lock.release()
 
-        return frame_tuple[0] * 1000000, TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3], frame_tuple[4])
+        return int(frame_tuple[0] * 1000000), TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3])
diff --git a/src/argaze/utils/replay_tobii_session.py b/src/argaze/utils/replay_tobii_session.py
index b911d09..87f043d 100644
--- a/src/argaze/utils/replay_tobii_session.py
+++ b/src/argaze/utils/replay_tobii_session.py
@@ -35,25 +35,38 @@ def main():
 
         # Access to timestamped gaze position data buffer
         tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp
-
         print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded')
 
-        # video and data replay loop
+        # Video and data replay loop
         try:
 
-            for frame_ts, frame in tobii_segment_video.frames():
+            # Iterate on video frames activating video / data synchronisation through vts data buffer
+            for video_ts, video_frame in tobii_segment_video.frames(tobii_segment_data.vts):
+
+                try:
+
+                    # Get closest gaze position before video timestamp and remove all gaze positions before
+                    closest_gaze_ts, closest_gaze_position = tobii_ts_gaze_positions.pop_first_until(video_ts)
+
+                    # Draw video synchronized gaze pointer
+                    pointer = (int(closest_gaze_position.gp[0] * video_frame.width), int(closest_gaze_position.gp[1] * video_frame.height))
+                    cv.circle(video_frame.matrix, pointer, 4, (0, 255, 255), -1)
+
+                # When expected values can't be found
+                except (KeyError, AttributeError, ValueError):
+                    pass
 
-                # close window using 'Esc' key
+                # Close window using 'Esc' key
                 if cv.waitKey(1) == 27:
                     break
 
-                cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix)
+                cv.imshow(f'Segment {tobii_segment.get_id()} video', video_frame.matrix)
 
-        # exit on 'ctrl+C' interruption
+        # Exit on 'ctrl+C' interruption
         except KeyboardInterrupt:
             pass
 
-        # stop frame display
+        # Stop frame display
         cv.destroyAllWindows()
 
 if __name__ == '__main__':