aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThéo de la Hogue2024-09-03 09:12:53 +0200
committerThéo de la Hogue2024-09-03 09:12:53 +0200
commit79f7f860a6e42297d0ffbb9df6298f1f29e9bcef (patch)
tree092db007d6a53ca05bea0c368bdff792be56e2af
parentb9854a924145db3869d481e81afcdb7550beddc5 (diff)
parent91a30dfadad1425c834f00e1022f7e42603af646 (diff)
downloadargaze-79f7f860a6e42297d0ffbb9df6298f1f29e9bcef.zip
argaze-79f7f860a6e42297d0ffbb9df6298f1f29e9bcef.tar.gz
argaze-79f7f860a6e42297d0ffbb9df6298f1f29e9bcef.tar.bz2
argaze-79f7f860a6e42297d0ffbb9df6298f1f29e9bcef.tar.xz
merging main and resolving conflict about performance in a use case.
-rw-r--r--src/argaze/utils/contexts/PupilLabsInvisible.py (renamed from src/argaze/utils/contexts/PupilLabs.py)7
-rw-r--r--src/argaze/utils/contexts/TobiiProGlasses3.py128
-rw-r--r--src/argaze/utils/demo/aruco_markers_pipeline.json24
-rw-r--r--src/argaze/utils/demo/gaze_analysis_pipeline.json2
-rw-r--r--src/argaze/utils/demo/pupillabs_invisible_live_stream_context.json6
-rw-r--r--src/argaze/utils/demo/pupillabs_live_stream_context.json6
-rw-r--r--src/argaze/utils/demo/tobii_g2_live_stream_context.json (renamed from src/argaze/utils/demo/tobii_live_stream_context.json)0
-rw-r--r--src/argaze/utils/demo/tobii_g3_live_stream_context.json6
8 files changed, 148 insertions, 31 deletions
diff --git a/src/argaze/utils/contexts/PupilLabs.py b/src/argaze/utils/contexts/PupilLabsInvisible.py
index 1bfb658..5c9a138 100644
--- a/src/argaze/utils/contexts/PupilLabs.py
+++ b/src/argaze/utils/contexts/PupilLabsInvisible.py
@@ -21,6 +21,7 @@ __license__ = "GPLv3"
import sys
import logging
import time
+
import threading
from dataclasses import dataclass
@@ -43,7 +44,7 @@ class LiveStream(ArFeatures.DataCaptureContext):
def __enter__(self):
- logging.info('Pupil-Labs Device connexion starts...')
+ logging.info('Pupil-Labs Invisible connexion starts...')
# Init timestamp
self.__start_time = time.time()
@@ -76,7 +77,7 @@ class LiveStream(ArFeatures.DataCaptureContext):
def __stream_gaze(self):
"""Stream gaze."""
- logging.debug('Stream gaze from Pupil Device')
+ logging.debug('Stream gaze from Pupil Invisible')
while self.is_running():
@@ -107,7 +108,7 @@ class LiveStream(ArFeatures.DataCaptureContext):
def __stream_video(self):
"""Stream video."""
- logging.debug('Stream video from Pupil Device')
+ logging.debug('Stream video from Pupil Invisible')
while self.is_running():
diff --git a/src/argaze/utils/contexts/TobiiProGlasses3.py b/src/argaze/utils/contexts/TobiiProGlasses3.py
new file mode 100644
index 0000000..a53c095
--- /dev/null
+++ b/src/argaze/utils/contexts/TobiiProGlasses3.py
@@ -0,0 +1,128 @@
+"""Handle network connection to Tobii Pro G3 devices.
+ Based on Tobii Realtime Python API.
+ g3pylib must be installed.
+"""
+
+"""
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+You should have received a copy of the GNU General Public License along with
+this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""
+
+__author__ = "Damien Mouratille"
+__credits__ = []
+__copyright__ = "Copyright 2024, Ecole Nationale de l'Aviation Civile (ENAC)"
+__license__ = "GPLv3"
+
+import sys
+import logging
+import time
+import dill
+import threading
+from dataclasses import dataclass
+import numpy
+import cv2
+import asyncio
+import os
+
+from argaze import ArFeatures, DataFeatures, GazeFeatures
+from argaze.utils import UtilsFeatures
+
+
+from g3pylib import connect_to_glasses
+
+
+class LiveStream(ArFeatures.DataCaptureContext):
+
+ @DataFeatures.PipelineStepInit
+ def __init__(self, **kwargs):
+
+ # Init DataCaptureContext class
+ super().__init__()
+
+ def __enter__(self):
+
+ logging.info('Tobii Pro G3 connexion starts...')
+
+ # Init timestamp
+ self.__start_time = time.time()
+
+ self.__loop = asyncio.new_event_loop()
+ self.__loop.run_until_complete(self.__stream_rtsp())
+
+ return self
+
+ async def __stream_rtsp(self):
+ """Stream video and gaze."""
+
+ logging.info('Stream gaze from Tobii Pro G3')
+
+ while self.is_running():
+
+ try:
+ async with connect_to_glasses.with_zeroconf(True,10000) as g3:
+ async with g3.stream_rtsp(scene_camera=True, gaze=True) as streams:
+ async with streams.gaze.decode() as gaze_stream, streams.scene_camera.decode() as scene_stream:
+ while True:
+ frame, frame_timestamp = await scene_stream.get()
+ gaze, gaze_timestamp = await gaze_stream.get()
+ while gaze_timestamp is None or frame_timestamp is None:
+ if frame_timestamp is None:
+ frame, frame_timestamp = await scene_stream.get()
+ if gaze_timestamp is None:
+ gaze, gaze_timestamp = await gaze_stream.get()
+ while gaze_timestamp < frame_timestamp:
+ gaze, gaze_timestamp = await gaze_stream.get()
+ while gaze_timestamp is None:
+ gaze, gaze_timestamp = await gaze_stream.get()
+
+ scene_frame = frame.to_ndarray(format="bgr24")
+
+ gaze_timestamp = int((gaze_timestamp - self.__start_time) * 1e3)
+
+ logging.debug('Gaze received at %i timestamp', gaze_timestamp)
+
+ # If given gaze data
+ if "gaze2d" in gaze:
+ gaze2d = gaze["gaze2d"]
+ # Convert rational (x,y) to pixel location (x,y)
+ h, w = scene_frame.shape[:2]
+ gaze_scene = (int(gaze2d[0] * w), int(gaze2d[1] * h))
+
+
+ self._process_gaze_position(
+ timestamp=gaze_timestamp,
+ x=gaze_scene[0],
+ y=gaze_scene[1])
+ else:
+ # Process empty gaze position
+ logging.debug('Not worn at %i timestamp', gaze_timestamp)
+
+ scene_timestamp = int((frame_timestamp - self.__start_time) * 1e3)
+
+ logging.debug('Video received at %i timestamp', scene_timestamp)
+
+ self._process_camera_image(
+ timestamp=scene_timestamp,
+ image=scene_frame)
+
+ except KeyboardInterrupt:
+ pass
+
+
+
+ @DataFeatures.PipelineStepExit
+ def __exit__(self, exception_type, exception_value, exception_traceback):
+
+ logging.debug('Tobii Pro G3 context stops...')
+
+ # Close data stream
+ self.stop()
+
diff --git a/src/argaze/utils/demo/aruco_markers_pipeline.json b/src/argaze/utils/demo/aruco_markers_pipeline.json
index 0681bc3..8221cec 100644
--- a/src/argaze/utils/demo/aruco_markers_pipeline.json
+++ b/src/argaze/utils/demo/aruco_markers_pipeline.json
@@ -1,7 +1,7 @@
{
"argaze.ArUcoMarker.ArUcoCamera.ArUcoCamera": {
"name": "Head-mounted camera",
- "size": [1920, 1080],
+ "size": [1088, 1080],
"copy_background_into_scenes_frames": true,
"aruco_detector": {
"dictionary": "DICT_APRILTAG_16h5",
@@ -56,7 +56,7 @@
},
"frames": {
"GrayRectangle": {
- "size": [1920, 1149],
+ "size": [1088, 1080],
"background": "frame_background.jpg",
"gaze_movement_identifier": {
"argaze.GazeAnalysis.DispersionThresholdIdentification.GazeMovementIdentifier": {
@@ -71,17 +71,12 @@
"argaze.GazeAnalysis.Basic.ScanPathAnalyzer": {},
"argaze.GazeAnalysis.KCoefficient.ScanPathAnalyzer": {},
"argaze.GazeAnalysis.NearestNeighborIndex.ScanPathAnalyzer": {
- "size": [1920, 1149]
+ "size": [1088, 1080]
},
"argaze.GazeAnalysis.ExploreExploitRatio.ScanPathAnalyzer": {
"short_fixation_duration_threshold": 0
}
},
- "observers": {
- "recorders.ScanPathAnalysisRecorder": {
- "path": "_export/records/scan_path_metrics.csv"
- }
- },
"layers": {
"demo_layer": {
"aoi_scene": "aoi_2d_scene.json",
@@ -101,11 +96,6 @@
"n_max": 3
},
"argaze.GazeAnalysis.Entropy.AOIScanPathAnalyzer":{}
- },
- "observers": {
- "recorders.AOIScanPathAnalysisRecorder": {
- "path": "_export/records/aoi_scan_path_metrics.csv"
- }
}
}
},
@@ -152,14 +142,6 @@
}
}
}
- },
- "observers": {
- "argaze.utils.UtilsFeatures.LookPerformanceRecorder": {
- "path": "_export/records/look_performance.csv"
- },
- "argaze.utils.UtilsFeatures.WatchPerformanceRecorder": {
- "path": "_export/records/watch_performance.csv"
- }
}
}
} \ No newline at end of file
diff --git a/src/argaze/utils/demo/gaze_analysis_pipeline.json b/src/argaze/utils/demo/gaze_analysis_pipeline.json
index 8b8212e..6e23321 100644
--- a/src/argaze/utils/demo/gaze_analysis_pipeline.json
+++ b/src/argaze/utils/demo/gaze_analysis_pipeline.json
@@ -1,7 +1,7 @@
{
"argaze.ArFeatures.ArFrame": {
"name": "GrayRectangle",
- "size": [1920, 1149],
+ "size": [1088, 1080],
"background": "frame_background.jpg",
"gaze_movement_identifier": {
"argaze.GazeAnalysis.DispersionThresholdIdentification.GazeMovementIdentifier": {
diff --git a/src/argaze/utils/demo/pupillabs_invisible_live_stream_context.json b/src/argaze/utils/demo/pupillabs_invisible_live_stream_context.json
new file mode 100644
index 0000000..3418de6
--- /dev/null
+++ b/src/argaze/utils/demo/pupillabs_invisible_live_stream_context.json
@@ -0,0 +1,6 @@
+{
+ "argaze.utils.contexts.PupilLabsInvisible.LiveStream" : {
+ "name": "PupilLabs Invisible",
+ "pipeline": "aruco_markers_pipeline.json"
+ }
+} \ No newline at end of file
diff --git a/src/argaze/utils/demo/pupillabs_live_stream_context.json b/src/argaze/utils/demo/pupillabs_live_stream_context.json
deleted file mode 100644
index bcb7263..0000000
--- a/src/argaze/utils/demo/pupillabs_live_stream_context.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "argaze.utils.contexts.PupilLabs.LiveStream" : {
- "name": "PupilLabs",
- "pipeline": "aruco_markers_pipeline.json"
- }
-} \ No newline at end of file
diff --git a/src/argaze/utils/demo/tobii_live_stream_context.json b/src/argaze/utils/demo/tobii_g2_live_stream_context.json
index 6950617..6950617 100644
--- a/src/argaze/utils/demo/tobii_live_stream_context.json
+++ b/src/argaze/utils/demo/tobii_g2_live_stream_context.json
diff --git a/src/argaze/utils/demo/tobii_g3_live_stream_context.json b/src/argaze/utils/demo/tobii_g3_live_stream_context.json
new file mode 100644
index 0000000..20f6ab1
--- /dev/null
+++ b/src/argaze/utils/demo/tobii_g3_live_stream_context.json
@@ -0,0 +1,6 @@
+{
+ "argaze.utils.contexts.TobiiProGlasses3.LiveStream" : {
+ "name": "Tobii Pro Glasses 3 live stream",
+ "pipeline": "aruco_markers_pipeline.json"
+ }
+} \ No newline at end of file