aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/argaze/DataStructures.py10
-rw-r--r--src/argaze/TobiiGlassesPro2/TobiiController.py331
-rw-r--r--src/argaze/TobiiGlassesPro2/TobiiData.py384
-rw-r--r--src/argaze/TobiiGlassesPro2/TobiiEntities.py108
-rw-r--r--src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py224
-rw-r--r--src/argaze/TobiiGlassesPro2/TobiiVideo.py184
-rw-r--r--src/argaze/TobiiGlassesPro2/__init__.py2
-rw-r--r--src/argaze/utils/README.md4
-rw-r--r--src/argaze/utils/calibrate_tobii_camera.py47
-rw-r--r--src/argaze/utils/display_tobii_gaze.py75
-rw-r--r--src/argaze/utils/export_tobii_segment_aruco_markers.py100
-rw-r--r--src/argaze/utils/live_tobii_aruco_detection.py (renamed from src/argaze/utils/track_aruco_rois_with_tobii_glasses.py)48
-rw-r--r--src/argaze/utils/live_tobii_session.py76
-rw-r--r--src/argaze/utils/replay_tobii_session.py61
-rw-r--r--src/argaze/utils/synchronise_timestamped_data.py50
15 files changed, 1134 insertions, 570 deletions
diff --git a/src/argaze/DataStructures.py b/src/argaze/DataStructures.py
index abaea62..4dab036 100644
--- a/src/argaze/DataStructures.py
+++ b/src/argaze/DataStructures.py
@@ -22,6 +22,12 @@ class DictObject():
def keys(self):
return list(self.__dict__.keys())[:-1]
+
+ def append(self, key, value):
+ __type_key = list(self.__dict__.keys())[-1]
+ __type_value = self.__dict__.pop(__type_key)
+ self.__dict__.update({key:value})
+ self.__dict__[__type_key] = __type_value
class TimeStampedBuffer(collections.OrderedDict):
"""Ordered dictionary to handle timestamped data.
@@ -57,6 +63,10 @@ class TimeStampedBuffer(collections.OrderedDict):
"""Easing FIFO access mode"""
return self.popitem(last=False)
+ def pop_last(self):
+ """Easing FIFO access mode"""
+ return self.popitem(last=True)
+
def export_as_json(self, filepath):
"""Write buffer content into a json file"""
try:
diff --git a/src/argaze/TobiiGlassesPro2/TobiiController.py b/src/argaze/TobiiGlassesPro2/TobiiController.py
index 7ab00cc..89dbc71 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiController.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiController.py
@@ -1,9 +1,14 @@
#!/usr/bin/env python
-import tobiiglassesctrl
+import datetime
-class TobiiController(tobiiglassesctrl.TobiiGlassesController):
- """As TobiiController inherits from TobiiGlassesPyController, here is its [code](https://github.com/ddetommaso/TobiiGlassesPyController/blob/master/tobiiglassesctrl/controller.py)."""
+from argaze.TobiiGlassesPro2 import TobiiNetworkInterface, TobiiData, TobiiVideo
+
+TOBII_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S+%f'
+TOBII_DATETIME_FORMAT_HUMREAD = '%d/%m/%Y %H:%M:%S'
+
+class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface):
+ """Handle Tobii glasses Pro 2 device using network interface."""
project_name = None
"""Project identifier."""
@@ -14,30 +19,322 @@ class TobiiController(tobiiglassesctrl.TobiiGlassesController):
calibration_id = None
"""Calibration identifier."""
- recording_id = None
- """Recording identifier."""
-
def __init__(self, ip_address, project_name, participant_id):
"""Create a project, a participant and start calibration."""
- super().__init__(ip_address, video_scene = True)
+ super().__init__(ip_address)
self.project_name = project_name
- self.project_id = super().create_project(self.project_name)
- self.participant_id = super().create_participant(self.project_id, self.project_name)
-
+ self.project_id = self.create_project(self.project_name)
+ self.participant_id = self.create_participant(self.project_id, self.project_name)
+
+ self.__recording_index = 0
+
+ self.__data_stream = None
+ self.__video_stream = None
+
+ super().wait_for_status('/api/system/status', 'sys_status', ['ok']) == 'ok'
+
+ def __get_current_datetime(self, timeformat=TOBII_DATETIME_FORMAT):
+ return datetime.datetime.now().replace(microsecond=0).strftime(timeformat)
+
def calibrate(self):
"""Start Tobii glasses calibration"""
input('Position Tobbi glasses calibration target then presse \'Enter\' to start calibration.')
- self.calibration_id = super().create_calibration(self.project_id, self.participant_id)
- super().start_calibration(self.calibration_id)
- if not super().wait_until_calibration_is_done(self.calibration_id):
- raise Error('Tobii calibration failed')
+ data = {
+ 'ca_project': self.project_id,
+ 'ca_type': 'default',
+ 'ca_participant': self.participant_id,
+ 'ca_created': self.__get_current_datetime()
+ }
+
+ json_data = super().post_request('/api/calibrations', data)
+
+ self.calibration_id = json_data['ca_id']
+
+ super().post_request('/api/calibrations/' + self.calibration_id + '/start')
+
+ status = super().wait_for_status('/api/calibrations/' + self.calibration_id + '/status', 'ca_state', ['calibrating', 'calibrated', 'stale', 'uncalibrated', 'failed'])
+
+ if status == 'uncalibrated' or status == 'stale' or status == 'failed':
+ raise Error(f'Tobii calibration {self.calibration_id} {status}')
+
+ # STREAMING FEATURES
+
+ def enable_data_stream(self):
+ """Enable Tobii Glasses Pro 2 data streaming."""
+
+ if self.__data_stream == None:
+ self.__data_stream = TobiiData.TobiiDataStream(self)
+
+ return self.__data_stream
+
+ def enable_video_stream(self):
+ """Enable Tobii Glasses Pro 2 video camera streaming."""
+
+ if self.__video_stream == None:
+ self.__video_stream = TobiiVideo.TobiiVideoStream(self)
+
+ return self.__video_stream
+
+ def start_streaming(self):
+
+ if self.__data_stream != None:
+ self.__data_stream.open()
+
+ if self.__video_stream != None:
+ self.__video_stream.open()
+
+ def stop_streaming(self):
+
+ if self.__data_stream != None:
+ self.__data_stream.close()
+
+ if self.__video_stream != None:
+ self.__video_stream.close()
+
+ # PROJECT FEATURES
+
+ def get_projects(self):
+ return super().get_request('/api/projects')
+
+ def get_project_id(self, project_name):
+
+ project_id = None
+ projects = super().get_request('/api/projects')
+
+ for project in projects:
+
+ try:
+ if project['pr_info']['Name'] == project_name:
+ project_id = project['pr_id']
+ except:
+ pass
+
+ return project_id
+
+ def create_project(self, projectname = 'DefaultProjectName'):
+
+ project_id = self.get_project_id(projectname)
+
+ if project_id is None:
+
+ data = {
+ 'pr_info' : {
+ 'CreationDate': self.__get_current_datetime(timeformat=TOBII_DATETIME_FORMAT_HUMREAD),
+ 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, projectname)),
+ 'Name': projectname
+ },
+ 'pr_created': self.__get_current_datetime()
+ }
+
+ json_data = super().post_request('/api/projects', data)
+
+ return json_data['pr_id']
+
+ else:
+
+ return project_id
+
+ # PARTICIPANT FEATURES
+
+ def create_participant(self, project_id, participant_name = 'DefaultUser', participant_notes = ''):
+
+ participant_id = self.get_participant_id(participant_name)
+ self.participant_name = participant_name
+
+ if participant_id is None:
+
+ data = {
+ 'pa_project': project_id,
+ 'pa_info': {
+ 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, self.participant_name)),
+ 'Name': self.participant_name,
+ 'Notes': participant_notes
+ },
+ 'pa_created': self.__get_current_datetime()
+ }
+
+ json_data = super().post_request('/api/participants', data)
+
+ return json_data['pa_id']
+
+ else:
+
+ return participant_id
+
+ def get_participant_id(self, participant_name):
+
+ participant_id = None
+ participants = super().get_request('/api/participants')
+
+ for participant in participants:
- def record(self):
+ try:
+ if participant['pa_info']['Name'] == participant_name:
+ participant_id = participant['pa_id']
+
+ except:
+ pass
+
+ return participant_id
+
+ def get_participants(self):
+ return super().get_request('/api/participants')
+
+ # RECORDING FEATURES
+
+ def __wait_for_recording_status(self, recording_id, status_array = ['init', 'starting', 'recording', 'pausing', 'paused', 'stopping', 'stopped', 'done', 'stale', 'failed']):
+ return super().wait_for_status('/api/recordings/' + recording_id + '/status', 'rec_state', status_array)
+
+ def create_recording(self, participant_id, recording_notes = ''):
+
+ self.__recording_index += 1
+ recording_name = f'Recording_{self.__recording_index}'
+
+ data = {
+ 'rec_participant': participant_id,
+ 'rec_info': {
+ 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, self.participant_name)),
+ 'Name': recording_name,
+ 'Notes': recording_notes},
+ 'rec_created': self.__get_current_datetime()
+ }
+
+ json_data = super().post_request('/api/recordings', data)
+
+ return json_data['rec_id']
+
+ def start_recording(self, recording_id):
"""Enable recording on the Tobii interface's SD Card"""
+
+ super().post_request('/api/recordings/' + recording_id + '/start')
+ if self.__wait_for_recording_status(recording_id, ['recording']) == 'recording':
+ return True
+
+ return False
+
+ def stop_recording(self, recording_id):
+ """Disable recording on the Tobii interface's SD Card"""
+
+ super().post_request('/api/recordings/' + recording_id + '/stop')
+ return self.__wait_for_recording_status(recording_id, ['done']) == "done"
+
+ def pause_recording(self, recording_id):
+ super().post_request('/api/recordings/' + recording_id + '/pause')
+ return self.__wait_for_recording_status(recording_id, ['paused']) == "paused"
+
+ def get_recording_status(self):
+ return self.get_status()['sys_recording']
+
+ def get_current_recording_id(self):
+ return self.get_recording_status()['rec_id']
+
+ def is_recording(self):
+
+ rec_status = self.get_recording_status()
+
+ if rec_status != {}:
+ if rec_status['rec_state'] == "recording":
+ return True
+
+ return False
+
+ def get_recordings(self):
+ return super().get_request('/api/recordings')
+
+ # MISC
+
+ def eject_sd(self):
+ super().get_request('/api/eject')
+
+ def get_battery_info(self):
+ return ( "Battery info = [ Level: %.2f %% - Remaining Time: %.2f s ]" % (float(self.get_battery_level()), float(self.get_battery_remaining_time())) )
+
+ def get_battery_level(self):
+ return self.get_battery_status()['level']
+
+ def get_battery_remaining_time(self):
+ return self.get_battery_status()['remaining_time']
+
+ def get_battery_status(self):
+ return self.get_status()['sys_battery']
+
+ def get_et_freq(self):
+ return self.get_configuration()['sys_et_freq']
+
+ def get_et_frequencies(self):
+ return self.get_status()['sys_et']['frequencies']
+
+ def identify(self):
+ super().get_request('/api/identify')
+
+ def get_address(self):
+ return self.address
+
+ def get_configuration(self):
+ return super().get_request('/api/system/conf')
+
+ def get_status(self):
+ return super().get_request('/api/system/status')
+
+ def get_storage_info(self):
+ return ( "Storage info = [ Remaining Time: %.2f s ]" % float(self.get_battery_remaining_time()) )
+
+ def get_storage_remaining_time(self):
+ return self.get_storage_status()['remaining_time']
+
+ def get_storage_status(self):
+ return self.get_status()['sys_storage']
+
+ def get_video_freq(self):
+ return self.get_configuration()['sys_sc_fps']
+
+ def send_custom_event(self, event_type, event_tag = ''):
+ data = {'type': event_type, 'tag': event_tag}
+ super().post_request('/api/events', data, wait_for_response=False)
+
+ def send_experimental_var(self, variable_name, variable_value):
+ self.send_custom_event('#%s#' % variable_name, variable_value)
+
+ def send_experimental_vars(self, variable_names_list, variable_values_list):
+ self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list))
+
+ def send_tobiipro_event(self, event_type, event_value):
+ self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value))
+
+ def set_et_freq_50(self):
+ data = {'sys_et_freq': 50}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_et_freq_100(self):
+ """May not be available. Check get_et_frequencies() first."""
+ data = {'sys_et_freq': 100}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_et_indoor_preset(self):
+ data = {'sys_sc_preset': 'Indoor'}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_et_outdoor_preset(self):
+ data = {'sys_ec_preset': 'ClearWeather'}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_video_auto_preset(self):
+ data = {'sys_sc_preset': 'Auto'}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_video_gaze_preset(self):
+ data = {'sys_sc_preset': 'GazeBasedExposure'}
+ json_data = super().post_request('/api/system/conf', data)
+
+ def set_video_freq_25(self):
+ data = {'sys_sc_fps': 25}
+ json_data = super().post_request('/api/system/conf/', data)
+
+ def set_video_freq_50(self):
+ data = {'sys_sc_fps': 50}
+ json_data = super().post_request('/api/system/conf/', data)
- self.recording_id = super().create_recording(self.participant_id)
- super().start_recording(self.recording_id)
diff --git a/src/argaze/TobiiGlassesPro2/TobiiData.py b/src/argaze/TobiiGlassesPro2/TobiiData.py
index 8e0a8b3..09afd33 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiData.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiData.py
@@ -1,317 +1,191 @@
#!/usr/bin/env python
import threading
+import uuid
+import json
import time
+import queue
-from argaze.TobiiGlassesPro2 import TobiiController
+from argaze import DataStructures
+from argaze.TobiiGlassesPro2 import TobiiNetworkInterface
-class TobiiDataThread(threading.Thread):
- """Handle data reception in a separate thread."""
+class TobiiSegmentData(DataStructures.DictObject):
+ """Handle Tobii Glasses Pro 2 segment data file."""
- def __init__(self, controller: TobiiController.TobiiController):
- """Initialise thread super class and prepare data reception."""
-
- threading.Thread.__init__(self)
+ def __init__(self, segment_data_path):
+ """Load segment data from segment directory then parse and register each recorded dataflow as a TimeStampedBuffer member of the TobiiSegmentData instance."""
- self.stop_event = threading.Event()
- self.read_lock = threading.Lock()
+ self.__segment_data_path = segment_data_path
+ self.__ts_start = 0
- self.controller = controller
+ ts_data_buffer_dict = {}
- self.fps = self.controller.get_et_freq()
- self.sleep = 1./self.fps
+ # define a decoder function
+ def decode(json_item):
- self.__ac_buffer = []
- self.__gy_buffer = []
- self.__gp_buffer = []
- self.__pts_buffer = []
+ # accept only valid data (e.g. with status value equal to 0)
+ if json_item.pop('s', -1) == 0:
- self.__start_ts = 0
+ # convert timestamp
+ ts = json_item.pop('ts')
- def __del__(self):
- pass
+ # keep first timestamp to offset all timestamps
+ if self.__ts_start == 0:
+ self.__ts_start = ts
- def __get_ac(self, data):
+ ts -= self.__ts_start
- ac_value = data['mems']['ac']['ac']
- ac_ts = data['mems']['ac']['ts']
- ac_data = {
- 'TIMESTAMP': ac_ts,
- 'TIME': (ac_ts - self.__start_ts) / 1000000.,
- 'X': ac_value[0],
- 'Y': ac_value[1],
- 'Z': ac_value[2]
- }
+ # ignore negative timestamp
+ if ts < 0:
+ return
- return ac_data
+ # convert json data into data object
+ data_object_type = '_'.join(json_item.keys())
+ data_object = DataStructures.DictObject(data_object_type, **json_item)
- def __get_gy(self, data):
+ # append a dedicated timestamped buffer for each data object type
+ if data_object.get_type() not in ts_data_buffer_dict.keys():
+ ts_data_buffer_dict[data_object.get_type()] = DataStructures.TimeStampedBuffer()
- gy_value = data['mems']['gy']['gy']
- gy_ts = data['mems']['gy']['ts']
- gy_data = {
- 'TIMESTAMP': gy_ts,
- 'TIME': (gy_ts - self.__start_ts) / 1000000.,
- 'X': gy_value[0],
- 'Y': gy_value[1],
- 'Z': gy_value[2]
- }
+ # store data object into the timestamped buffer dedicated to its type
+ ts_data_buffer_dict[data_object.get_type()][ts] = data_object
- return gy_data
+ # start loading
+ with gzip.open(self.__segment_data_path) as f:
- def __get_gp(self, data):
+ for item in f:
+ json.loads(item.decode('utf-8'), object_hook=decode)
- gp_value = data['gp']['gp']
- gp_ts = data['gp']['ts']
- gp_data = {
- 'TIMESTAMP': gp_ts,
- 'TIME': (gp_ts - self.__start_ts) / 1000000.,
- 'X': gp_value[0],
- 'Y': gp_value[1]
- }
+ super().__init__(type(self).__name__, **ts_data_buffer_dict)
- return gp_data
+ def keys(self):
+ """Get all registered data keys"""
+ return list(self.__dict__.keys())[2:-1]
- def __get_pts(self, data):
+ def get_path(self):
+ return self.__segment_data_path
- pts_value = data['pts']['pts']
- pts_ts = data['pts']['ts']
- pts_data = {
- 'TIMESTAMP': pts_ts,
- 'TIME': (pts_ts - self.__start_ts) / 1000000.,
- 'PTS': pts_value
- }
+class TobiiDataStream(threading.Thread):
+ """Capture Tobii Glasses Pro 2 data stream in separate thread."""
- return pts_data
+ def __init__(self, network_interface: TobiiNetworkInterface.TobiiNetworkInterface):
+ """Initialise thread super class as a deamon dedicated to data reception."""
- def run(self):
- """Data reception function."""
+ threading.Thread.__init__(self)
+ threading.Thread.daemon = True
+
+ self.__network = network_interface
+ self.__data_socket = self.__network.make_socket()
- while not self.stop_event.isSet():
+ self.__data_queue = queue.Queue()
- time.sleep(self.sleep)
+ self.__stop_event = threading.Event()
+ self.__read_lock = threading.Lock()
- self.read_lock.acquire()
+ self.__sleep = 1. / self.__network.get_et_freq()
- data = self.controller.get_data()
+ # prepare keep alive message
+ self.__keep_alive_msg = "{\"type\": \"live.data.unicast\", \"key\": \""+ str(uuid.uuid4()) +"\", \"op\": \"start\"}"
+ self.__keep_alive_thread = threading.Thread(target = self.__keep_alive)
+ self.__keep_alive_thread.daemon = True
- # store only timestamped datas
- if 'pts' in data:
+ def __del__(self):
+ """Stop data reception before destruction."""
- pts_data = data['pts']
+ self.close()
- if 'pts' in pts_data:
+ def __keep_alive(self):
+ """Maintain connection."""
- ac_ts = data['mems']['ac']['ts']
- gy_ts = data['mems']['gy']['ts']
- gp_ts = data['gp']['ts']
- pts_ts = pts_data['ts']
+ while not self.__stop_event.isSet():
- # get start timestamp
- if self.__start_ts == 0:
+ self.__network.send_keep_alive_msg(self.__data_socket, self.__keep_alive_msg)
- # ignore -1 timestamp
- valid_ts = []
- for ts in [ac_ts, gy_ts, gp_ts, pts_ts]:
- if ts > 0:
- valid_ts.append(ts)
+ time.sleep(1)
- self.__start_ts = min(valid_ts)
- #print(f'Tobii Data Frame: __start_ts = {self.__start_ts}')
+ def open(self):
+ """Start data reception."""
- #print(f'Tobii Data Frame: ac_ts = {ac_ts}, gy_ts = {gy_ts}, gp_ts = {gp_ts}, pts_ts = {pts_ts}')
+ self.__keep_alive_thread.start()
+ threading.Thread.start(self)
- # ignore -1 timestamp and filter repetitions
+ def close(self):
+ """Stop data reception definitively."""
- if ac_ts != -1:
- if len(self.__ac_buffer) == 0:
- self.__ac_buffer.append(self.__get_ac(data))
- elif ac_ts != self.__ac_buffer[-1]['TIMESTAMP']:
- self.__ac_buffer.append(self.__get_ac(data))
+ self.__stop_event.set()
- if gy_ts != -1:
- if len(self.__gy_buffer) == 0:
- self.__gy_buffer.append(self.__get_gy(data))
- elif gy_ts != self.__gy_buffer[-1]['TIMESTAMP']:
- self.__gy_buffer.append(self.__get_gy(data))
+ threading.Thread.join(self.__keep_alive_thread)
+ threading.Thread.join(self)
- if gp_ts != -1:
- if len(self.__gp_buffer) == 0:
- self.__gp_buffer.append(self.__get_gp(data))
- elif gp_ts != self.__gp_buffer[-1]['TIMESTAMP']:
- self.__gp_buffer.append(self.__get_gp(data))
+ self.__data_socket.close()
- if pts_ts != -1:
- if len(self.__pts_buffer) == 0:
- self.__pts_buffer.append(self.__get_pts(data))
- elif pts_ts != self.__pts_buffer[-1]['TIMESTAMP']:
- self.__pts_buffer.append(self.__get_pts(data))
+ def run(self):
+ """Store received data into a queue for further reading."""
- self.read_lock.release()
+ while not self.__stop_event.isSet():
- def read_accelerometer_data(self, timestamp: int = -1):
- """Get accelerometer data at a given timestamp.
- **Returns:** accelerometer dictionary
- ```
- {
- 'TIMESTAMP': int,
- 'TIME': int,
- 'X': float,
- 'Y': float,
- 'Z': float
- }
- ```
- """
+ # wait
+ time.sleep(self.__sleep)
- if len(self.__ac_buffer):
+ # lock data queue access
+ self.__read_lock.acquire()
- self.read_lock.acquire()
-
- # TODO : find closest timestamp data
- ac_data = self.__ac_buffer[-1].copy()
+ # write in data queue
+ data = self.__network.grab_data(self.__data_socket)
+ self.__data_queue.put(data)
- self.read_lock.release()
+ # unlock data queue access
+ self.__read_lock.release()
- return ac_data
+ def read(self):
- else:
+ # create a dictionary of timestamped data buffers
+ ts_data_buffer_dict = DataStructures.DictObject('TobiiDataStream', **{})
- return {}
+ # if the data acquisition thread is not running
+ if self.__stop_event.isSet():
+ return ts_data_buffer_dict
- def read_accelerometer_buffer(self):
- """Get accelerometer data buffer.
- **Returns:** accelerometer dictionary array"""
+ # lock data queue access
+ self.__read_lock.acquire()
- self.read_lock.acquire()
-
- ac_buffer = self.__ac_buffer.copy()
+ # read data queue
+ while not self.__data_queue.empty():
- self.read_lock.release()
+ data = self.__data_queue.get()
- return ac_buffer
+ json_item = json.loads(data.decode('utf-8'))
- def read_gyroscope_data(self, timestamp: int = -1):
- """Get gyroscope data at a given timestamp.
- **Returns:** gyroscope dictionary
- ```
- {
- 'TIMESTAMP': int,
- 'TIME': int,
- 'X': float,
- 'Y': float,
- 'Z': float
- }
- ```
- """
+ # accept only valid data (e.g. with status value equal to 0)
+ if json_item.pop('s', -1) == 0:
- if len(self.__gy_buffer):
+ # convert timestamp
+ ts = json_item.pop('ts')
- self.read_lock.acquire()
+ #print(f'json_item at {ts}: {json_item}')
+ '''
+ # keep first timestamp to offset all timestamps
+ if self.__ts_start == 0:
+ self.__ts_start = ts
- # TODO : find closest timestamp data
- gy_data = self.__gy_buffer[-1].copy()
-
- self.read_lock.release()
-
- return gy_data
-
- else:
-
- return {}
-
- def read_gyroscope_buffer(self):
- """Get gyroscope data buffer.
- **Returns:** gyroscope dictionary array"""
-
- self.read_lock.acquire()
-
- gy_buffer = self.__gy_buffer.copy()
-
- self.read_lock.release()
-
- return gy_buffer
-
- def read_gaze_data(self, timestamp: int = -1):
- """Get gaze data at a given timestamp.
- **Returns:** gaze dictionary
- ```
- {
- 'TIMESTAMP': int,
- 'TIME': int,
- 'X': float,
- 'Y': float
- }
- ```
- """
-
- if len(self.__gp_buffer):
-
- self.read_lock.acquire()
+ ts -= self.__ts_start
- # TODO : find closest timestamp data
- gp_data = self.__gp_buffer[-1].copy()
-
- self.read_lock.release()
-
- return gp_data
-
- else:
-
- return {}
-
- def read_gaze_buffer(self):
- """Get gaze data buffer.
- **Returns:** gaze dictionary array"""
-
- self.read_lock.acquire()
-
- gp_buffer = self.__gp_buffer.copy()
-
- self.read_lock.release()
-
- return gp_buffer
-
- def read_pts_data(self, timestamp: int = -1):
- """Get Presentation Time Stamp (pts) data at a given timestamp.
- **Returns:** pts dictionary
- ```
- {
- 'TIMESTAMP': int,
- 'TIME': int,
- 'PTS': int
- }
- ```
- """
-
- if len(self.__pts_buffer):
-
- self.read_lock.acquire()
+ # ignore negative timestamp
+ if ts < 0:
+ break
+ '''
+ # convert json data into data object
+ data_object_type = '_'.join(json_item.keys())
+ data_object = DataStructures.DictObject(data_object_type, **json_item)
+
+ # append a dedicated timestamped buffer for each data object type
+ if data_object.get_type() not in ts_data_buffer_dict.keys():
+ ts_data_buffer_dict.append(data_object.get_type(), DataStructures.TimeStampedBuffer())
+
+ # store data object into the timestamped buffer dedicated to its type
+ ts_data_buffer_dict[data_object.get_type()][ts] = data_object
- # TODO : find closest timestamp data
- pts_data = self.__pts_buffer[-1].copy()
+ # unlock data queue access
+ self.__read_lock.release()
- self.read_lock.release()
-
- return pts_data
-
- else:
-
- return {}
-
- def read_pts_buffer(self):
- """Get Presentation Time Stamp (pts) data buffer.
- **Returns:** pts dictionary array"""
-
- self.read_lock.acquire()
-
- pts_buffer = self.__pts_buffer.copy()
-
- self.read_lock.release()
-
- return pts_buffer
-
- def stop(self):
- """Stop data reception definitively."""
-
- self.stop_event.set()
- threading.Thread.join(self)
+ return ts_data_buffer_dict
diff --git a/src/argaze/TobiiGlassesPro2/TobiiEntities.py b/src/argaze/TobiiGlassesPro2/TobiiEntities.py
index e3b0d9b..9dc41bc 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiEntities.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiEntities.py
@@ -26,114 +26,6 @@ TOBII_SEGMENT_INFO_FILENAME = "segment.json"
TOBII_SEGMENT_VIDEO_FILENAME = "fullstream.mp4"
TOBII_SEGMENT_DATA_FILENAME = "livedata.json.gz"
-class TobiiSegmentData(DataStructures.DictObject):
- """Handle Tobii Glasses Pro 2 segment data file."""
-
- def __init__(self, segment_data_path):
- """Load segment data from segment directory then parse and register each recorded dataflow as a TimeStampedBuffer member of the TobiiSegmentData instance."""
-
- self.__segment_data_path = segment_data_path
- self.__ts_start = 0
-
- ts_data_buffer_dict = {}
-
- # define a decoder function
- def decode(json_item):
-
- # accept only valid data (e.g. with status value equal to 0)
- if json_item.pop('s', -1) == 0:
-
- # convert timestamp
- ts = json_item.pop('ts')
-
- # keep first timestamp to offset all timestamps
- if self.__ts_start == 0:
- self.__ts_start = ts
-
- ts -= self.__ts_start
-
- # ignore negative timestamp
- if ts < 0:
- return
-
- # convert json data into data object
- data_object_type = '_'.join(json_item.keys())
- data_object = DataStructures.DictObject(data_object_type, **json_item)
-
- # append a dedicated timestamped buffer for each data object type
- if data_object.get_type() not in ts_data_buffer_dict.keys():
- ts_data_buffer_dict[data_object.get_type()] = DataStructures.TimeStampedBuffer()
-
- # store data object into the timestamped buffer dedicated to its type
- ts_data_buffer_dict[data_object.get_type()][ts] = data_object
-
- # start loading
- with gzip.open(self.__segment_data_path) as f:
-
- for item in f:
- json.loads(item.decode('utf-8'), object_hook=decode)
-
- super().__init__(type(self).__name__, **ts_data_buffer_dict)
-
- def keys(self):
- """Get all registered data keys"""
- return list(self.__dict__.keys())[2:-1]
-
- def get_path(self):
- return self.__segment_data_path
-
-class TobiiVideoFrame(DataStructures.DictObject):
- """Define tobii video frame"""
-
- def __init__(self, matrix, width, height, pts):
-
- super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts})
-
-class TobiiSegmentVideo():
- """Handle Tobii Glasses Pro 2 segment video file."""
-
- def __init__(self, segment_video_path):
- """Load segment video from segment directory"""
-
- self.__segment_video_path = segment_video_path
- self.__container = av.open(self.__segment_video_path)
- self.__stream = self.__container.streams.video[0]
-
- self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH))
- self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT))
-
- def get_path(self):
- return self.__segment_video_path
-
- def get_duration(self):
- return float(self.__stream.duration * self.__stream.time_base)
-
- def get_frame_number(self):
- return self.__stream.frames
-
- def get_width(self):
- return self.__width
-
- def get_height(self):
- return self.__height
-
- def frames(self):
- return self.__iter__()
-
- def __iter__(self):
-
- # start decoding
- self.__container.decode(self.__stream)
-
- return self
-
- def __next__(self):
-
- frame = self.__container.decode(self.__stream).__next__()
-
- # return micro second timestamp and frame data
- return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts)
-
class TobiiSegment:
"""Handle Tobii Glasses Pro 2 segment info."""
diff --git a/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py b/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py
new file mode 100644
index 0000000..13c0b9a
--- /dev/null
+++ b/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py
@@ -0,0 +1,224 @@
+import logging
+import sys
+import socket
+import threading
+import json
+import time
+
+# python2 backwards compatibility for errors
+if sys.version_info[0] < 3:
+ class ConnectionError(BaseException):
+ pass
+
+try:
+ import netifaces
+ TOBII_DISCOVERY_ALLOWED = True
+except:
+ TOBII_DISCOVERY_ALLOWED = False
+
+try:
+ from urllib.parse import urlparse, urlencode
+ from urllib.request import urlopen, Request
+ from urllib.error import URLError, HTTPError
+
+except ImportError:
+ from urlparse import urlparse
+ from urllib import urlencode
+ from urllib2 import urlopen, Request, HTTPError, URLError
+
+socket.IPPROTO_IPV6 = 41
+
+class TobiiNetworkInterface():
+ """Handle network connection to Tobii glasses Pro 2 device.
+ It is a major rewrite of [tobiiglassesctrl/controller.py](https://github.com/ddetommaso/TobiiGlassesPyController/blob/master/tobiiglassesctrl/controller.py)."""
+
+ def __init__(self, address = None):
+
+ self.udpport = 49152
+ self.address = address
+ self.iface_name = None
+
+ if self.address is None:
+
+ data, address = self.__discover_device()
+
+ if address is None:
+ raise ConnectionError("No device found using discovery process")
+ else:
+ try:
+ self.address = data["ipv4"]
+ except:
+ self.address = address
+
+ if "%" in self.address:
+ if sys.platform == "win32":
+ self.address,self.iface_name = self.address.split("%")
+ else:
+ self.iface_name = self.address.split("%")[1]
+
+ if ':' in self.address:
+ self.base_url = 'http://[%s]' % self.address
+ else:
+ self.base_url = 'http://' + self.address
+
+ self.__peer = (self.address, self.udpport)
+
+ def make_socket(self):
+
+ iptype = socket.AF_INET
+
+ if ':' in self.__peer[0]:
+ iptype = socket.AF_INET6
+
+ res = socket.getaddrinfo(self.__peer[0], self.__peer[1], socket.AF_UNSPEC, socket.SOCK_DGRAM, 0, socket.AI_PASSIVE)
+ family, socktype, proto, canonname, sockaddr = res[0]
+ new_socket = socket.socket(family, socktype, proto)
+
+ new_socket.settimeout(5.0)
+
+ try:
+ if iptype == socket.AF_INET6:
+ new_socket.setsockopt(socket.SOL_SOCKET, 25, 1)
+
+ except socket.error as e:
+ if e.errno == 1:
+ logging.warning("Binding to a network interface is permitted only for root users.")
+
+ return new_socket
+
+ def connect(self, timeout = None):
+
+ return self.wait_for_status('/api/system/status', 'sys_status', ['ok']) == 'ok'
+
+ def __discover_device(self):
+
+ if TOBII_DISCOVERY_ALLOWED == False:
+ logging.error("Device discovery is not available due to a missing dependency (netifaces)")
+ exit(1)
+
+ logging.debug("Looking for a Tobii Pro Glasses 2 device ...")
+
+ MULTICAST_ADDR = 'ff02::1'
+ PORT = 13006
+
+ for i in netifaces.interfaces():
+
+ if netifaces.AF_INET6 in netifaces.ifaddresses(i).keys():
+
+ if "%" in netifaces.ifaddresses(i)[netifaces.AF_INET6][0]['addr']:
+
+ if_name = netifaces.ifaddresses(i)[netifaces.AF_INET6][0]['addr'].split("%")[1]
+ if_idx = socket.getaddrinfo(MULTICAST_ADDR + "%" + if_name, PORT, socket.AF_INET6, socket.SOCK_DGRAM)[0][4][3]
+
+ s6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
+ s6.settimeout(30.0)
+ s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, if_idx)
+ s6.bind(('::', PORT))
+
+ PORT_OUT = PORT if sys.platform == 'win32' or sys.platform == 'darwin' else PORT + 1
+
+ try:
+ discover_json = '{"type":"discover"}'
+ s6.sendto(discover_json.encode('utf-8'), (MULTICAST_ADDR, PORT_OUT))
+
+ logging.debug("Discover request sent to %s on interface %s " % ( str((MULTICAST_ADDR, PORT_OUT)),if_name) )
+ logging.debug("Waiting for a reponse from the device ...")
+
+ data, address = s6.recvfrom(1024)
+ jdata = json.loads(data.decode('utf-8'))
+
+ logging.debug("From: " + address[0] + " " + str(data))
+ logging.debug("Tobii Pro Glasses found with address: [%s]" % address[0])
+
+ addr = address[0]
+
+ if sys.version_info.major == 3 and sys.version_info.minor >= 8:
+ addr = address[0] + '%' + if_name
+
+ return (jdata, addr)
+
+ except:
+ logging.debug("No device found on interface %s" % if_name)
+
+ logging.debug("The discovery process did not find any device!")
+
+ return (None, None)
+
+ def get_request(self, api_action):
+
+ url = self.base_url + api_action
+ res = urlopen(url).read()
+ try:
+ data = json.loads(res.decode('utf-8'))
+ except json.JSONDecodeError:
+ data = None
+ return data
+
+ def post_request(self, api_action, data=None, wait_for_response=True):
+
+ url = self.base_url + api_action
+ req = Request(url)
+ req.add_header('Content-Type', 'application/json')
+ data = json.dumps(data)
+
+ logging.debug("Sending JSON: " + str(data))
+
+ if wait_for_response is False:
+ threading.Thread(target=urlopen, args=(req, data.encode('utf-8'),)).start()
+ return None
+
+ response = urlopen(req, data.encode('utf-8'))
+ res = response.read()
+
+ logging.debug("Response: " + str(res))
+
+ try:
+ res = json.loads(res.decode('utf-8'))
+
+ except:
+ pass
+
+ return res
+
+ def send_keep_alive_msg(self, socket, msg):
+
+ res = socket.sendto(msg.encode('utf-8'), self.__peer)
+
+ def grab_data(self, socket):
+
+ try:
+ data, address = socket.recvfrom(1024)
+ return data
+
+ except TimeoutError:
+
+ logging.error("A timeout occurred while receiving data")
+
+ def wait_for_status(self, api_action, key, values, timeout = None):
+
+ url = self.base_url + api_action
+ running = True
+
+ while running:
+
+ req = Request(url)
+ req.add_header('Content-Type', 'application/json')
+
+ try:
+
+ response = urlopen(req, None, timeout = timeout)
+
+ except URLError as e:
+
+ logging.error(e.reason)
+ return -1
+
+ data = response.read()
+ json_data = json.loads(data.decode('utf-8'))
+
+ if json_data[key] in values:
+ running = False
+
+ time.sleep(1)
+
+ return json_data[key]
diff --git a/src/argaze/TobiiGlassesPro2/TobiiVideo.py b/src/argaze/TobiiGlassesPro2/TobiiVideo.py
index b927b7f..748967a 100644
--- a/src/argaze/TobiiGlassesPro2/TobiiVideo.py
+++ b/src/argaze/TobiiGlassesPro2/TobiiVideo.py
@@ -1,96 +1,156 @@
#!/usr/bin/env python
import threading
+import uuid
+import time
+import copy
+
+from argaze import DataStructures
+from argaze.TobiiGlassesPro2 import TobiiNetworkInterface
import av
import numpy
-class TobiiVideoThread(threading.Thread):
- """Handle video camera stream capture in a separate thread."""
+class TobiiVideoFrame(DataStructures.DictObject):
+ """Define tobii video frame"""
+
+ def __init__(self, matrix, width, height, pts):
+
+ super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts})
+
+class TobiiVideoSegment():
+ """Handle Tobii Glasses Pro 2 segment video file."""
+
+ def __init__(self, segment_video_path):
+ """Load segment video from segment directory"""
+
+ self.__segment_video_path = segment_video_path
+ self.__container = av.open(self.__segment_video_path)
+ self.__stream = self.__container.streams.video[0]
+
+ self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH))
+ self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT))
+
+ def get_path(self):
+ return self.__segment_video_path
+
+ def get_duration(self):
+ return float(self.__stream.duration * self.__stream.time_base)
+
+ def get_frame_number(self):
+ return self.__stream.frames
+
+ def get_width(self):
+ return self.__width
+
+ def get_height(self):
+ return self.__height
+
+ def frames(self):
+ return self.__iter__()
+
+ def __iter__(self):
+
+ # start decoding
+ self.__container.decode(self.__stream)
+
+ return self
+
+ def __next__(self):
- def __init__(self, controller):
- """Initialise thread super class and prepare camera video stream reception."""
+ frame = self.__container.decode(self.__stream).__next__()
+
+ # return micro second timestamp and frame data
+ return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts)
+
+class TobiiVideoStream(threading.Thread):
+ """Capture Tobii Glasses Pro 2 video camera stream."""
+
+ def __init__(self, network_interface: TobiiNetworkInterface.TobiiNetworkInterface):
+ """Initialise video stream reception."""
threading.Thread.__init__(self)
- self.stop_event = threading.Event()
- self.read_lock = threading.Lock()
+ threading.Thread.daemon = True
- self.controller = controller
+ self.__network = network_interface
+ self.__video_socket = self.__network.make_socket()
- self.fps = self.controller.get_video_freq()
+ self.__stop_event = threading.Event()
+ self.__read_lock = threading.Lock()
- self.read_lock.acquire()
+ self.__frame_tuple = None
- self.__frame = numpy.zeros((1, 1, 3), numpy.uint8)
- self.__width = 0
- self.__height = 0
- self.__pts_buffer = []
+ self.__sleep = 1. / self.__network.get_video_freq()
- self.read_lock.release()
+ # prepare keep alive message
+ self.__keep_alive_msg = "{\"type\": \"live.video.unicast\",\"key\": \""+ str(uuid.uuid4()) +"_video\", \"op\": \"start\"}"
+ self.__keep_alive_thread = threading.Timer(0, self.__keep_alive)
+ self.__keep_alive_thread.daemon = True
def __del__(self):
- pass
+ """Stop data reception before destruction."""
- def run(self):
- """Video camera stream capture function."""
+ self.close()
- # start Tobii glasses stream capture
- self.__container = av.open(f'rtsp://{self.controller.get_address()}:8554/live/scene', options={'rtsp_transport': 'tcp'})
- self.__stream = self.__container.streams.video[0]
-
- for f in self.__container.decode(self.__stream):
+ def __keep_alive(self):
+ """Maintain connection."""
- if self.stop_event.isSet():
- break
+ while not self.__stop_event.isSet():
+
+ self.__network.send_keep_alive_msg(self.__video_socket, self.__keep_alive_msg)
+
+ time.sleep(1)
- self.read_lock.acquire()
-
- self.__frame = f.to_ndarray(format='bgr24')
- self.__width = f.width
- self.__height = f.height
- self.__pts_buffer.append({'TIME':f.time, 'PTS': f.pts})
-
- self.read_lock.release()
+ def open(self):
+ """Start data reception."""
- def read(self) :
- """Read video frame.
- **Returns:** frame, frame width, frame height, frame time, frame pts."""
+ self.__keep_alive_thread.start()
+ threading.Thread.start(self)
- # if stopped, return blank frame
- if self.stop_event.isSet():
- return numpy.zeros((1, 1, 3), numpy.uint8)
+ def close(self):
+ """Stop data reception definitively."""
- # else
- self.read_lock.acquire()
+ self.__stop_event.set()
- frame_copy = self.__frame.copy()
- width_copy = self.__width
- height_copy = self.__height
+ threading.Thread.join(self.__keep_alive_thread)
+ threading.Thread.join(self)
- if len(self.__pts_buffer):
- time_copy = self.__pts_buffer[-1]['TIME']
- pts_copy = self.__pts_buffer[-1]['PTS']
- else:
- time_copy = -1
- pts_copy = -1
+ self.__video_socket.close()
- self.read_lock.release()
+ def run(self):
+ """Store frame for further reading."""
- return frame_copy, width_copy, height_copy, time_copy, pts_copy
+ container = av.open(f'rtsp://{self.__network.get_address()}:8554/live/scene', options={'rtsp_transport': 'tcp'})
+ stream = container.streams.video[0]
- def read_pts_buffer(self):
- """Get Presentation Time Stamp data buffer."""
+ for frame in container.decode(stream):
- self.read_lock.acquire()
-
- pts_buffer = self.__pts_buffer.copy()
+ # quit if the video acquisition thread have been stopped
+ if self.__stop_event.isSet():
+ break
- self.read_lock.release()
+ # lock frame access
+ self.__read_lock.acquire()
- return pts_buffer
+ # store frame time, matrix, width, height and pts into a tuple
+ self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts)
- def stop(self):
- """Stop video camera stream capture definitively."""
+ # unlock frame access
+ self.__read_lock.release()
- self.stop_event.set()
- threading.Thread.join(self)
+ def read(self):
+
+ # if the video acquisition thread have been stopped or isn't started
+ if self.__stop_event.isSet() or self.__frame_tuple == None:
+ return -1, TobiiVideoFrame(numpy.zeros((1, 1, 3), numpy.uint8), 1, 1, -1)
+
+ # lock frame access
+ self.__read_lock.acquire()
+
+ # copy frame tuple
+ frame_tuple = copy.deepcopy(self.__frame_tuple)
+
+ # unlock frame access
+ self.__read_lock.release()
+
+ return frame_tuple[0] * 1000000, TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3], frame_tuple[4])
diff --git a/src/argaze/TobiiGlassesPro2/__init__.py b/src/argaze/TobiiGlassesPro2/__init__.py
index 3c92c1f..329402c 100644
--- a/src/argaze/TobiiGlassesPro2/__init__.py
+++ b/src/argaze/TobiiGlassesPro2/__init__.py
@@ -2,4 +2,4 @@
.. include:: README.md
"""
__docformat__ = "restructuredtext"
-__all__ = ['TobiiEntities','TobiiController', 'TobiiData', 'TobiiVideo'] \ No newline at end of file
+__all__ = ['TobiiEntities', 'TobiiNetworkInterface', 'TobiiController', 'TobiiData', 'TobiiVideo'] \ No newline at end of file
diff --git a/src/argaze/utils/README.md b/src/argaze/utils/README.md
index 461d82d..ffe2c16 100644
--- a/src/argaze/utils/README.md
+++ b/src/argaze/utils/README.md
@@ -30,10 +30,10 @@ python ./src/argaze/utils/export_calibration_board.py 7 5 5 3 -o export
python ./src/argaze/utils/calibrate_tobii_camera.py 7 5 5 3 -t IP_ADDRESS -o export/tobii_camera.json
```
-- Display Tobii Glasses Pro 2 gaze and camera video stream (replace IP_ADDRESS)
+- Display Tobii Glasses Pro 2 camera video stream (replace IP_ADDRESS) and gaze
```
-python ./src/argaze/utils/display_tobii_gaze.py -t IP_ADDRESS
+python ./src/argaze/utils/live_tobii_session.py -t IP_ADDRESS
```
- Record a Tobii Glasses Pro 2 'Test' session for a participant '1' on Tobii interface's SD card (replace IP_ADDRESS).
diff --git a/src/argaze/utils/calibrate_tobii_camera.py b/src/argaze/utils/calibrate_tobii_camera.py
index 93751ba..0381f75 100644
--- a/src/argaze/utils/calibrate_tobii_camera.py
+++ b/src/argaze/utils/calibrate_tobii_camera.py
@@ -26,30 +26,29 @@ def main():
parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
parser.add_argument('columns', metavar='COLS_NUMBER', type=int, default=7, help='number of columns')
parser.add_argument('rows', metavar='ROWS_NUMBER', type=int, default=5, help='number of rows')
- parser.add_argument('square_size', metavar='SQUARE_SIZE', type=int, default=5, help='square size (cm)')
- parser.add_argument('marker_size', metavar='MARKER_SIZE', type=int, default=5, help='marker size (cm)')
+ parser.add_argument('square_size', metavar='SQUARE_SIZE', type=float, default=5, help='square size (cm)')
+ parser.add_argument('marker_size', metavar='MARKER_SIZE', type=float, default=3, help='marker size (cm)')
parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip')
parser.add_argument('-o', '--output', metavar='OUT', type=str, default='.', help='destination filepath')
parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_4X4_50', help='aruco marker dictionnary')
args = parser.parse_args()
- # create tobii controller
+ # Create tobii controller
tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1)
- # create tobii video thread
- tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller)
- tobii_video_thread.start()
+ # Enable tobii video stream
+ tobii_video_stream = tobii_controller.enable_video_stream()
- # create aruco camera
+ # Create aruco camera
aruco_camera = ArUcoCamera.ArUcoCamera()
- # create aruco board
+ # Create aruco board
aruco_board = ArUcoBoard.ArUcoBoard(args.dictionary, args.columns, args.rows, args.square_size, args.marker_size)
- # create aruco tracker
+ # Create aruco tracker
aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, args.marker_size, aruco_camera)
- # start tobii glasses streaming
+ # Start tobii glasses streaming
tobii_controller.start_streaming()
print("Camera calibration starts")
@@ -64,32 +63,35 @@ def main():
# capture loop
try:
- # wait 1ms between each frame until 'Esc' key is pressed
- while cv.waitKey(1) != 27:
+ while tobii_video_stream.is_alive():
# capture frame with a full displayed board
- frame, frame_width, frame_height, frame_time, frame_pts = tobii_video_thread.read()
+ video_ts, video_frame = tobii_video_stream.read()
# track all markers in the board
- aruco_tracker.track_board(frame, aruco_board, expected_markers_number)
+ aruco_tracker.track_board(video_frame.matrix, aruco_board, expected_markers_number)
# draw only markers
- aruco_tracker.draw(frame)
+ aruco_tracker.draw(video_frame.matrix)
# draw current calibration data count
- cv.putText(frame, f'Capture: {aruco_camera.get_calibration_data_count()}', (50, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv.LINE_AA)
- cv.imshow('Tobii Camera Calibration', frame)
+ cv.putText(video_frame.matrix, f'Capture: {aruco_camera.get_calibration_data_count()}', (50, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv.LINE_AA)
+ cv.imshow('Tobii Camera Calibration', video_frame.matrix)
# if all board corners are detected
if aruco_tracker.get_board_corners_number() == expected_corners_number:
# draw board corners to notify a capture is done
- aruco_tracker.draw_board(frame)
+ aruco_tracker.draw_board(video_frame.matrix)
# append data
aruco_camera.store_calibration_data(aruco_tracker.get_board_corners(), aruco_tracker.get_board_corners_ids())
- cv.imshow('Tobii Camera Calibration', frame)
+ cv.imshow('Tobii Camera Calibration', video_frame.matrix)
+
+ # close window using 'Esc' key
+ if cv.waitKey(1) == 27:
+ break
# exit on 'ctrl+C' interruption
except KeyboardInterrupt:
@@ -98,14 +100,11 @@ def main():
# stop frame display
cv.destroyAllWindows()
- # stop tobii objects
- tobii_video_thread.stop()
-
+ # Stop tobii glasses streaming
tobii_controller.stop_streaming()
- tobii_controller.close()
print('\nCalibrating camera...')
- aruco_camera.calibrate(aruco_board, frame_width, frame_height)
+ aruco_camera.calibrate(aruco_board, video_frame.width, video_frame.height)
print('\nCalibration succeeded!')
print(f'\nRMS:\n{aruco_camera.get_rms()}')
diff --git a/src/argaze/utils/display_tobii_gaze.py b/src/argaze/utils/display_tobii_gaze.py
deleted file mode 100644
index 18b89f8..0000000
--- a/src/argaze/utils/display_tobii_gaze.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import os, time
-
-from argaze.TobiiGlassesPro2 import *
-
-import cv2 as cv
-import numpy
-
-def main():
- """
- Capture video camera and display gaze point
- """
-
- # manage arguments
- parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
- parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip')
-
- args = parser.parse_args()
-
- # create tobii controller
- tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1)
-
- # calibrate tobii glasses
- tobii_controller.calibrate()
-
- # create tobii data thread
- tobii_data_thread = TobiiData.TobiiDataThread(tobii_controller)
- tobii_data_thread.start()
-
- # create tobii video thread
- tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller)
- tobii_video_thread.start()
-
- # start tobii glasses streaming
- tobii_controller.start_streaming()
-
- # display loop
- try:
-
- # wait 1ms between each frame until 'Esc' key is pressed
- while cv.waitKey(1) != 27:
-
- frame, frame_width, frame_height, frame_time, pts = tobii_video_thread.read()
-
- # draw tobii gaze
- # TODO : sync gaze data according frame pts
- gp_data = tobii_data_thread.read_gaze_data(pts)
- if 'TIMESTAMP' in gp_data:
- pointer = (int(gp_data['X'] * frame_width), int(gp_data['Y'] * frame_height))
- cv.circle(frame, pointer, 4, (0, 255, 255), -1)
- else:
- pointer = (0, 0)
-
- # display frame
- cv.imshow('Tobii Camera Record', frame)
-
- # exit on 'ctrl+C' interruption
- except KeyboardInterrupt:
- pass
-
- # stop frame display
- cv.destroyAllWindows()
-
- # stop tobii objects
- tobii_video_thread.stop()
- tobii_data_thread.stop()
-
- tobii_controller.stop_streaming()
- tobii_controller.close()
-
-if __name__ == '__main__':
-
- main() \ No newline at end of file
diff --git a/src/argaze/utils/export_tobii_segment_aruco_markers.py b/src/argaze/utils/export_tobii_segment_aruco_markers.py
new file mode 100644
index 0000000..11c5c1b
--- /dev/null
+++ b/src/argaze/utils/export_tobii_segment_aruco_markers.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+import argparse
+import bisect
+
+from argaze import GazeFeatures
+from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo
+from argaze.ArUcoMarkers import *
+from argaze.RegionOfInterest import *
+
+import numpy
+
+import cv2 as cv
+
+def main():
+ """
+ Replay Tobii segment video
+ """
+
+ # manage arguments
+ parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
+ parser.add_argument('-s', '--segment_path', metavar='SEGMENT_PATH', type=str, default=None, help='segment path')
+ args = parser.parse_args()
+
+ if args.segment_path != None:
+
+ # Load a tobii segment
+ tobii_segment = TobiiEntities.TobiiSegment(args.segment_path)
+
+ # create aruco camera
+ aruco_camera = ArUcoCamera.ArUcoCamera()
+ aruco_camera.load_calibration_file('/Users/Robotron/Developpements/ArGaze/export/tobii_camera.json')
+
+ # create aruco tracker
+ aruco_tracker = ArUcoTracker.ArUcoTracker('DICT_ARUCO_ORIGINAL', 7.5, aruco_camera) # aruco dictionary, marker length (cm), camera
+
+ # Load a tobii segment video
+ tobii_segment_video = tobii_segment.load_video()
+ print(f'Video duration: {tobii_segment_video.get_duration()}, frame number: {tobii_segment_video.get_frame_number()}, width: {tobii_segment_video.get_width()}, height: {tobii_segment_video.get_height()}')
+
+ # create ROIs 3D scene
+ roi3D_scene = ROI3DScene.ROI3DScene()
+ roi3D_scene.load('/Users/Robotron/Developpements/ArGaze/export/test.obj')
+
+ # replay loop
+ try:
+
+ last_ts = 0
+ for frame_ts, frame in tobii_segment_video.frames():
+
+ if frame_ts > last_ts:
+
+ delay = int((frame_ts - last_ts) / 1000)
+
+ if cv.waitKey(delay) == 27:
+ break
+
+ # track markers with pose estimation and draw them
+ aruco_tracker.track(frame.matrix)
+ aruco_tracker.draw(frame.matrix)
+
+ # project 3D scenes related to each aruco markers
+ if aruco_tracker.get_markers_number():
+
+ for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()):
+
+ # TODO : select different 3D scenes depending on aruco id
+
+ marker_rotation = aruco_tracker.get_marker_rotation(i)
+ marker_translation = aruco_tracker.get_marker_translation(i)
+
+ roi3D_scene.set_rotation(marker_rotation)
+ roi3D_scene.set_translation(marker_translation)
+
+ # zero distorsion matrix
+ D0 = numpy.asarray([0.0, 0.0, 0.0, 0.0, 0.0])
+
+ # DON'T APPLY CAMERA DISTORSION : it projects points which are far from the frame into it
+ # This hack isn't realistic but as the gaze will mainly focus on centered ROI, where the distorsion is low, it is acceptable.
+ roi2D_scene = roi3D_scene.project(aruco_camera.get_K(), D0)
+
+ # check if gaze is inside 2D rois
+ #roi2D_scene.inside(pointer)
+
+ # draw 2D rois
+ roi2D_scene.draw(frame.matrix)
+
+ cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix)
+ last_ts = frame_ts
+
+ # exit on 'ctrl+C' interruption
+ except KeyboardInterrupt:
+ pass
+
+ # stop frame display
+ cv.destroyAllWindows()
+
+if __name__ == '__main__':
+
+ main() \ No newline at end of file
diff --git a/src/argaze/utils/track_aruco_rois_with_tobii_glasses.py b/src/argaze/utils/live_tobii_aruco_detection.py
index 334a6bb..82837d6 100644
--- a/src/argaze/utils/track_aruco_rois_with_tobii_glasses.py
+++ b/src/argaze/utils/live_tobii_aruco_detection.py
@@ -3,6 +3,7 @@
import argparse
import os
+from argaze.TobiiGlassesPro2 import *
from argaze.ArUcoMarkers import ArUcoTracker, ArUcoCamera
from argaze.RegionOfInterest import *
from argaze.TobiiGlassesPro2 import *
@@ -20,45 +21,43 @@ def main():
Export all collected datas into an export folder for further analysis.
"""
- # manage arguments
+ # Manage arguments
parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip')
parser.add_argument('-c', '--camera_calibration', metavar='CAM_CALIB', type=str, default='tobii_camera.json', help='json camera calibration filepath')
parser.add_argument('-s', '--roi_scene', metavar='ROI_SCENE', type=str, default='roi3D_scene.obj', help='obj roi scene filepath')
parser.add_argument('-o', '--output', metavar='OUT', type=str, default='.', help='destination path')
- parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_4X4_50', help='aruco marker dictionnary')
+ parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_ARUCO_ORIGINAL', help='aruco marker dictionnary')
parser.add_argument('-m', '--marker_size', metavar='MKR', type=int, default=6, help='aruco marker size (cm)')
args = parser.parse_args()
- # create tobii controller
+ # Create tobii controller
tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1)
- # calibrate tobii glasses
- tobii_controller.calibrate()
+ # Calibrate tobii glasses
+ #tobii_controller.calibrate()
- # create tobii data thread
- tobii_data_thread = TobiiData.TobiiDataThread(tobii_controller)
- tobii_data_thread.start()
+ # Create tobii data stream
+ #tobii_data_stream = TobiiData.TobiiDataStream(tobii_controller)
- # create tobii video thread
- tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller)
- tobii_video_thread.start()
+ # Create tobii video stream
+ tobii_video_stream = TobiiVideo.TobiiVideoStream(tobii_controller)
# create aruco camera
aruco_camera = ArUcoCamera.ArUcoCamera()
aruco_camera.load_calibration_file(args.camera_calibration)
- # create aruco tracker
- aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, 6, aruco_camera) # aruco dictionaries, marker length (cm), camera
+ # Create aruco tracker
+ aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, 7.5, aruco_camera) # aruco dictionary, marker length (cm), camera
- # create ROIs 3D scene
+ # Create ROIs 3D scene
roi3D_scene = ROI3DScene.ROI3DScene()
roi3D_scene.load(args.roi_scene)
- # start tobii glasses streaming
+ # Start tobii glasses streaming
tobii_controller.start_streaming()
- # process video frames
+ # Process video frames
frame_time = 0
last_frame_time = 0
roi2D_buffer = []
@@ -67,14 +66,15 @@ def main():
# tracking loop
try:
- # wait 1ms between each frame until 'Esc' key is pressed
- while cv.waitKey(1) != 27:
-
- frame, frame_width, frame_height, frame_time, pts = tobii_video_thread.read()
+ for frame_ts, frame in tobii_video_stream.frames():
+ # close window using 'Esc' key
+ if cv.waitKey(1) == 27:
+ break
+
# draw tobii gaze
# TODO : sync gaze data according frame pts
- gp_data = tobii_data_thread.read_gaze_data(pts)
+ gp_data = tobii_data_stream.read_gaze_data(pts)
if 'TIMESTAMP' in gp_data:
pointer = (int(gp_data['X'] * frame_width), int(gp_data['Y'] * frame_height))
cv.circle(frame, pointer, 4, (0, 255, 255), -1)
@@ -137,12 +137,8 @@ def main():
cv.destroyAllWindows()
last_frame_time = frame_time
- # stop tobii objects
- tobii_video_thread.stop()
- tobii_data_thread.stop()
-
+ # stop tobii glasses streaming
tobii_controller.stop_streaming()
- tobii_controller.close()
# create a pandas DataFrame for each buffer
ac_dataframe = pandas.DataFrame(tobii_data_thread.read_accelerometer_buffer(), columns=['TIMESTAMP', 'TIME', 'X', 'Y', 'Z'])
diff --git a/src/argaze/utils/live_tobii_session.py b/src/argaze/utils/live_tobii_session.py
new file mode 100644
index 0000000..0181e23
--- /dev/null
+++ b/src/argaze/utils/live_tobii_session.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+import argparse
+import os, time
+
+from argaze import GazeFeatures
+from argaze.TobiiGlassesPro2 import *
+
+import cv2 as cv
+import numpy
+
+def main():
+ """
+ Capture video camera and display gaze point
+ """
+
+ # Manage arguments
+ parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
+ parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.12', help='tobii glasses ip')
+
+ args = parser.parse_args()
+
+ # Create tobii controller
+ tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1)
+
+ # Calibrate tobii glasses
+ tobii_controller.calibrate()
+
+ # Enable tobii data stream
+ tobii_data_stream = tobii_controller.enable_data_stream()
+
+ # Enable tobii video stream
+ tobii_video_stream = tobii_controller.enable_video_stream()
+
+ # Start streaming
+ tobii_controller.start_streaming()
+
+ # Live video stream capture loop
+ try:
+
+ while tobii_video_stream.is_alive():
+
+ video_ts, video_frame = tobii_video_stream.read()
+
+ try:
+
+ # get last gaze position
+ last_ts, last_gaze_position = tobii_data_stream.read().gidx_l_gp.pop_last()
+
+ # Draw tobii gaze pointer
+ pointer = (int(last_gaze_position.gp[0] * video_frame.width), int(last_gaze_position.gp[1] * video_frame.height))
+ cv.circle(video_frame.matrix, pointer, 4, (0, 255, 255), -1)
+
+ # when gidx_l_gp key not received during last frame
+ except (KeyError, AttributeError):
+ pass
+
+ # close window using 'Esc' key
+ if cv.waitKey(1) == 27:
+ break
+
+ cv.imshow(f'Live Tobii Camera', video_frame.matrix)
+
+ # exit on 'ctrl+C' interruption
+ except KeyboardInterrupt:
+ pass
+
+ # Stop frame display
+ cv.destroyAllWindows()
+
+ # Stop streaming
+ tobii_controller.stop_streaming()
+
+if __name__ == '__main__':
+
+ main() \ No newline at end of file
diff --git a/src/argaze/utils/replay_tobii_session.py b/src/argaze/utils/replay_tobii_session.py
new file mode 100644
index 0000000..b911d09
--- /dev/null
+++ b/src/argaze/utils/replay_tobii_session.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import argparse
+import bisect
+
+from argaze import GazeFeatures
+from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo, TobiiData
+
+import numpy
+
+import cv2 as cv
+
+def main():
+ """
+ Replay Tobii segment video
+ """
+
+ # manage arguments
+ parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0])
+ parser.add_argument('-s', '--segment_path', metavar='SEGMENT_PATH', type=str, default=None, help='segment path')
+ args = parser.parse_args()
+
+ if args.segment_path != None:
+
+ # Load a tobii segment
+ tobii_segment = TobiiEntities.TobiiSegment(args.segment_path)
+
+ # Load a tobii segment video
+ tobii_segment_video = tobii_segment.load_video()
+ print(f'Video duration: {tobii_segment_video.get_duration()}, frame number: {tobii_segment_video.get_frame_number()}, width: {tobii_segment_video.get_width()}, height: {tobii_segment_video.get_height()}')
+
+ # Load a tobii segment data
+ tobii_segment_data = tobii_segment.load_data()
+ print(f'Data keys: {tobii_segment_data.keys()}')
+
+ # Access to timestamped gaze position data buffer
+ tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp
+
+ print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded')
+
+ # video and data replay loop
+ try:
+
+ for frame_ts, frame in tobii_segment_video.frames():
+
+ # close window using 'Esc' key
+ if cv.waitKey(1) == 27:
+ break
+
+ cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix)
+
+ # exit on 'ctrl+C' interruption
+ except KeyboardInterrupt:
+ pass
+
+ # stop frame display
+ cv.destroyAllWindows()
+
+if __name__ == '__main__':
+
+ main() \ No newline at end of file
diff --git a/src/argaze/utils/synchronise_timestamped_data.py b/src/argaze/utils/synchronise_timestamped_data.py
new file mode 100644
index 0000000..0c22072
--- /dev/null
+++ b/src/argaze/utils/synchronise_timestamped_data.py
@@ -0,0 +1,50 @@
+
+
+
+
+
+
+
+ # Synchronise video and gaze
+ vts_ts, vts = tobii_vts.pop_first()
+ vts_offset = (vts_ts - vts.vts) / 1000
+
+ print(f'Init >>> vts_offset = {vts_offset}')
+
+ last_gaze_position_index = -1
+ closest_gaze_position_ts, closest_gaze_position = None, None
+
+ last_fixation_index = -1
+ closest_fixation_ts, closest_fixation = None, None
+
+ for frame_ts, frame in tobii_segment_video.frames():
+
+ frame_ts = frame_ts / 1000
+
+ if frame_ts > vts.vts / 1000:
+ if len(tobii_vts) > 0:
+ vts_ts, vts = tobii_vts.pop_first()
+ vts_offset = (vts_ts - vts.vts) / 1000
+ print(f'{frame_ts / 1000} >>> New vts_offset = {vts_offset / 1000}')
+
+ # Find closest gaze position
+ closest_index = bisect.bisect_left(list(generic_ts_gaze_positions.keys()), frame_ts + vts_offset) - 1
+ if closest_index > last_gaze_position_index:
+
+ # pop data until closest_index
+ while last_gaze_position_index <= closest_index:
+ closest_gaze_position_ts, closest_gaze_position = generic_ts_gaze_positions.pop_first()
+ last_gaze_position_index += 1
+
+ print(f'{frame_ts / 1000} *** New closest gaze position: index = {closest_index}, ts = {closest_gaze_position_ts / 1000}, {closest_gaze_position}')
+
+ # Find closest fixation
+ closest_index = bisect.bisect_left(list(fixation_analyser.fixations.keys()), frame_ts + vts_offset) - 1
+ if closest_index > last_fixation_index:
+
+ # pop data until closest_index
+ while last_fixation_index <= closest_index:
+ closest_ts, closest_fixation = fixation_analyser.fixations.pop_first()
+ last_fixation_index += 1
+
+ print(f'{frame_ts / 1000} /// New closest fixation: index= {closest_index}, ts = {closest_ts / 1000}, {closest_fixation}') \ No newline at end of file