diff options
| -rw-r--r-- | src/argaze/DataStructures.py | 10 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/TobiiController.py | 331 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/TobiiData.py | 384 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/TobiiEntities.py | 108 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py | 224 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/TobiiVideo.py | 184 | ||||
| -rw-r--r-- | src/argaze/TobiiGlassesPro2/__init__.py | 2 | ||||
| -rw-r--r-- | src/argaze/utils/README.md | 4 | ||||
| -rw-r--r-- | src/argaze/utils/calibrate_tobii_camera.py | 47 | ||||
| -rw-r--r-- | src/argaze/utils/display_tobii_gaze.py | 75 | ||||
| -rw-r--r-- | src/argaze/utils/export_tobii_segment_aruco_markers.py | 100 | ||||
| -rw-r--r-- | src/argaze/utils/live_tobii_aruco_detection.py (renamed from src/argaze/utils/track_aruco_rois_with_tobii_glasses.py) | 48 | ||||
| -rw-r--r-- | src/argaze/utils/live_tobii_session.py | 76 | ||||
| -rw-r--r-- | src/argaze/utils/replay_tobii_session.py | 61 | ||||
| -rw-r--r-- | src/argaze/utils/synchronise_timestamped_data.py | 50 |
15 files changed, 1134 insertions, 570 deletions
diff --git a/src/argaze/DataStructures.py b/src/argaze/DataStructures.py index abaea62..4dab036 100644 --- a/src/argaze/DataStructures.py +++ b/src/argaze/DataStructures.py @@ -22,6 +22,12 @@ class DictObject(): def keys(self): return list(self.__dict__.keys())[:-1] + + def append(self, key, value): + __type_key = list(self.__dict__.keys())[-1] + __type_value = self.__dict__.pop(__type_key) + self.__dict__.update({key:value}) + self.__dict__[__type_key] = __type_value class TimeStampedBuffer(collections.OrderedDict): """Ordered dictionary to handle timestamped data. @@ -57,6 +63,10 @@ class TimeStampedBuffer(collections.OrderedDict): """Easing FIFO access mode""" return self.popitem(last=False) + def pop_last(self): + """Easing FIFO access mode""" + return self.popitem(last=True) + def export_as_json(self, filepath): """Write buffer content into a json file""" try: diff --git a/src/argaze/TobiiGlassesPro2/TobiiController.py b/src/argaze/TobiiGlassesPro2/TobiiController.py index 7ab00cc..89dbc71 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiController.py +++ b/src/argaze/TobiiGlassesPro2/TobiiController.py @@ -1,9 +1,14 @@ #!/usr/bin/env python -import tobiiglassesctrl +import datetime -class TobiiController(tobiiglassesctrl.TobiiGlassesController): - """As TobiiController inherits from TobiiGlassesPyController, here is its [code](https://github.com/ddetommaso/TobiiGlassesPyController/blob/master/tobiiglassesctrl/controller.py).""" +from argaze.TobiiGlassesPro2 import TobiiNetworkInterface, TobiiData, TobiiVideo + +TOBII_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S+%f' +TOBII_DATETIME_FORMAT_HUMREAD = '%d/%m/%Y %H:%M:%S' + +class TobiiController(TobiiNetworkInterface.TobiiNetworkInterface): + """Handle Tobii glasses Pro 2 device using network interface.""" project_name = None """Project identifier.""" @@ -14,30 +19,322 @@ class TobiiController(tobiiglassesctrl.TobiiGlassesController): calibration_id = None """Calibration identifier.""" - recording_id = None - """Recording identifier.""" - def __init__(self, ip_address, project_name, participant_id): """Create a project, a participant and start calibration.""" - super().__init__(ip_address, video_scene = True) + super().__init__(ip_address) self.project_name = project_name - self.project_id = super().create_project(self.project_name) - self.participant_id = super().create_participant(self.project_id, self.project_name) - + self.project_id = self.create_project(self.project_name) + self.participant_id = self.create_participant(self.project_id, self.project_name) + + self.__recording_index = 0 + + self.__data_stream = None + self.__video_stream = None + + super().wait_for_status('/api/system/status', 'sys_status', ['ok']) == 'ok' + + def __get_current_datetime(self, timeformat=TOBII_DATETIME_FORMAT): + return datetime.datetime.now().replace(microsecond=0).strftime(timeformat) + def calibrate(self): """Start Tobii glasses calibration""" input('Position Tobbi glasses calibration target then presse \'Enter\' to start calibration.') - self.calibration_id = super().create_calibration(self.project_id, self.participant_id) - super().start_calibration(self.calibration_id) - if not super().wait_until_calibration_is_done(self.calibration_id): - raise Error('Tobii calibration failed') + data = { + 'ca_project': self.project_id, + 'ca_type': 'default', + 'ca_participant': self.participant_id, + 'ca_created': self.__get_current_datetime() + } + + json_data = super().post_request('/api/calibrations', data) + + self.calibration_id = json_data['ca_id'] + + super().post_request('/api/calibrations/' + self.calibration_id + '/start') + + status = super().wait_for_status('/api/calibrations/' + self.calibration_id + '/status', 'ca_state', ['calibrating', 'calibrated', 'stale', 'uncalibrated', 'failed']) + + if status == 'uncalibrated' or status == 'stale' or status == 'failed': + raise Error(f'Tobii calibration {self.calibration_id} {status}') + + # STREAMING FEATURES + + def enable_data_stream(self): + """Enable Tobii Glasses Pro 2 data streaming.""" + + if self.__data_stream == None: + self.__data_stream = TobiiData.TobiiDataStream(self) + + return self.__data_stream + + def enable_video_stream(self): + """Enable Tobii Glasses Pro 2 video camera streaming.""" + + if self.__video_stream == None: + self.__video_stream = TobiiVideo.TobiiVideoStream(self) + + return self.__video_stream + + def start_streaming(self): + + if self.__data_stream != None: + self.__data_stream.open() + + if self.__video_stream != None: + self.__video_stream.open() + + def stop_streaming(self): + + if self.__data_stream != None: + self.__data_stream.close() + + if self.__video_stream != None: + self.__video_stream.close() + + # PROJECT FEATURES + + def get_projects(self): + return super().get_request('/api/projects') + + def get_project_id(self, project_name): + + project_id = None + projects = super().get_request('/api/projects') + + for project in projects: + + try: + if project['pr_info']['Name'] == project_name: + project_id = project['pr_id'] + except: + pass + + return project_id + + def create_project(self, projectname = 'DefaultProjectName'): + + project_id = self.get_project_id(projectname) + + if project_id is None: + + data = { + 'pr_info' : { + 'CreationDate': self.__get_current_datetime(timeformat=TOBII_DATETIME_FORMAT_HUMREAD), + 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, projectname)), + 'Name': projectname + }, + 'pr_created': self.__get_current_datetime() + } + + json_data = super().post_request('/api/projects', data) + + return json_data['pr_id'] + + else: + + return project_id + + # PARTICIPANT FEATURES + + def create_participant(self, project_id, participant_name = 'DefaultUser', participant_notes = ''): + + participant_id = self.get_participant_id(participant_name) + self.participant_name = participant_name + + if participant_id is None: + + data = { + 'pa_project': project_id, + 'pa_info': { + 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, self.participant_name)), + 'Name': self.participant_name, + 'Notes': participant_notes + }, + 'pa_created': self.__get_current_datetime() + } + + json_data = super().post_request('/api/participants', data) + + return json_data['pa_id'] + + else: + + return participant_id + + def get_participant_id(self, participant_name): + + participant_id = None + participants = super().get_request('/api/participants') + + for participant in participants: - def record(self): + try: + if participant['pa_info']['Name'] == participant_name: + participant_id = participant['pa_id'] + + except: + pass + + return participant_id + + def get_participants(self): + return super().get_request('/api/participants') + + # RECORDING FEATURES + + def __wait_for_recording_status(self, recording_id, status_array = ['init', 'starting', 'recording', 'pausing', 'paused', 'stopping', 'stopped', 'done', 'stale', 'failed']): + return super().wait_for_status('/api/recordings/' + recording_id + '/status', 'rec_state', status_array) + + def create_recording(self, participant_id, recording_notes = ''): + + self.__recording_index += 1 + recording_name = f'Recording_{self.__recording_index}' + + data = { + 'rec_participant': participant_id, + 'rec_info': { + 'EagleId': str(uuid.uuid5(uuid.NAMESPACE_DNS, self.participant_name)), + 'Name': recording_name, + 'Notes': recording_notes}, + 'rec_created': self.__get_current_datetime() + } + + json_data = super().post_request('/api/recordings', data) + + return json_data['rec_id'] + + def start_recording(self, recording_id): """Enable recording on the Tobii interface's SD Card""" + + super().post_request('/api/recordings/' + recording_id + '/start') + if self.__wait_for_recording_status(recording_id, ['recording']) == 'recording': + return True + + return False + + def stop_recording(self, recording_id): + """Disable recording on the Tobii interface's SD Card""" + + super().post_request('/api/recordings/' + recording_id + '/stop') + return self.__wait_for_recording_status(recording_id, ['done']) == "done" + + def pause_recording(self, recording_id): + super().post_request('/api/recordings/' + recording_id + '/pause') + return self.__wait_for_recording_status(recording_id, ['paused']) == "paused" + + def get_recording_status(self): + return self.get_status()['sys_recording'] + + def get_current_recording_id(self): + return self.get_recording_status()['rec_id'] + + def is_recording(self): + + rec_status = self.get_recording_status() + + if rec_status != {}: + if rec_status['rec_state'] == "recording": + return True + + return False + + def get_recordings(self): + return super().get_request('/api/recordings') + + # MISC + + def eject_sd(self): + super().get_request('/api/eject') + + def get_battery_info(self): + return ( "Battery info = [ Level: %.2f %% - Remaining Time: %.2f s ]" % (float(self.get_battery_level()), float(self.get_battery_remaining_time())) ) + + def get_battery_level(self): + return self.get_battery_status()['level'] + + def get_battery_remaining_time(self): + return self.get_battery_status()['remaining_time'] + + def get_battery_status(self): + return self.get_status()['sys_battery'] + + def get_et_freq(self): + return self.get_configuration()['sys_et_freq'] + + def get_et_frequencies(self): + return self.get_status()['sys_et']['frequencies'] + + def identify(self): + super().get_request('/api/identify') + + def get_address(self): + return self.address + + def get_configuration(self): + return super().get_request('/api/system/conf') + + def get_status(self): + return super().get_request('/api/system/status') + + def get_storage_info(self): + return ( "Storage info = [ Remaining Time: %.2f s ]" % float(self.get_battery_remaining_time()) ) + + def get_storage_remaining_time(self): + return self.get_storage_status()['remaining_time'] + + def get_storage_status(self): + return self.get_status()['sys_storage'] + + def get_video_freq(self): + return self.get_configuration()['sys_sc_fps'] + + def send_custom_event(self, event_type, event_tag = ''): + data = {'type': event_type, 'tag': event_tag} + super().post_request('/api/events', data, wait_for_response=False) + + def send_experimental_var(self, variable_name, variable_value): + self.send_custom_event('#%s#' % variable_name, variable_value) + + def send_experimental_vars(self, variable_names_list, variable_values_list): + self.send_custom_event('@%s@' % str(variable_names_list), str(variable_values_list)) + + def send_tobiipro_event(self, event_type, event_value): + self.send_custom_event('JsonEvent', "{'event_type': '%s','event_value': '%s'}" % (event_type, event_value)) + + def set_et_freq_50(self): + data = {'sys_et_freq': 50} + json_data = super().post_request('/api/system/conf', data) + + def set_et_freq_100(self): + """May not be available. Check get_et_frequencies() first.""" + data = {'sys_et_freq': 100} + json_data = super().post_request('/api/system/conf', data) + + def set_et_indoor_preset(self): + data = {'sys_sc_preset': 'Indoor'} + json_data = super().post_request('/api/system/conf', data) + + def set_et_outdoor_preset(self): + data = {'sys_ec_preset': 'ClearWeather'} + json_data = super().post_request('/api/system/conf', data) + + def set_video_auto_preset(self): + data = {'sys_sc_preset': 'Auto'} + json_data = super().post_request('/api/system/conf', data) + + def set_video_gaze_preset(self): + data = {'sys_sc_preset': 'GazeBasedExposure'} + json_data = super().post_request('/api/system/conf', data) + + def set_video_freq_25(self): + data = {'sys_sc_fps': 25} + json_data = super().post_request('/api/system/conf/', data) + + def set_video_freq_50(self): + data = {'sys_sc_fps': 50} + json_data = super().post_request('/api/system/conf/', data) - self.recording_id = super().create_recording(self.participant_id) - super().start_recording(self.recording_id) diff --git a/src/argaze/TobiiGlassesPro2/TobiiData.py b/src/argaze/TobiiGlassesPro2/TobiiData.py index 8e0a8b3..09afd33 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiData.py +++ b/src/argaze/TobiiGlassesPro2/TobiiData.py @@ -1,317 +1,191 @@ #!/usr/bin/env python import threading +import uuid +import json import time +import queue -from argaze.TobiiGlassesPro2 import TobiiController +from argaze import DataStructures +from argaze.TobiiGlassesPro2 import TobiiNetworkInterface -class TobiiDataThread(threading.Thread): - """Handle data reception in a separate thread.""" +class TobiiSegmentData(DataStructures.DictObject): + """Handle Tobii Glasses Pro 2 segment data file.""" - def __init__(self, controller: TobiiController.TobiiController): - """Initialise thread super class and prepare data reception.""" - - threading.Thread.__init__(self) + def __init__(self, segment_data_path): + """Load segment data from segment directory then parse and register each recorded dataflow as a TimeStampedBuffer member of the TobiiSegmentData instance.""" - self.stop_event = threading.Event() - self.read_lock = threading.Lock() + self.__segment_data_path = segment_data_path + self.__ts_start = 0 - self.controller = controller + ts_data_buffer_dict = {} - self.fps = self.controller.get_et_freq() - self.sleep = 1./self.fps + # define a decoder function + def decode(json_item): - self.__ac_buffer = [] - self.__gy_buffer = [] - self.__gp_buffer = [] - self.__pts_buffer = [] + # accept only valid data (e.g. with status value equal to 0) + if json_item.pop('s', -1) == 0: - self.__start_ts = 0 + # convert timestamp + ts = json_item.pop('ts') - def __del__(self): - pass + # keep first timestamp to offset all timestamps + if self.__ts_start == 0: + self.__ts_start = ts - def __get_ac(self, data): + ts -= self.__ts_start - ac_value = data['mems']['ac']['ac'] - ac_ts = data['mems']['ac']['ts'] - ac_data = { - 'TIMESTAMP': ac_ts, - 'TIME': (ac_ts - self.__start_ts) / 1000000., - 'X': ac_value[0], - 'Y': ac_value[1], - 'Z': ac_value[2] - } + # ignore negative timestamp + if ts < 0: + return - return ac_data + # convert json data into data object + data_object_type = '_'.join(json_item.keys()) + data_object = DataStructures.DictObject(data_object_type, **json_item) - def __get_gy(self, data): + # append a dedicated timestamped buffer for each data object type + if data_object.get_type() not in ts_data_buffer_dict.keys(): + ts_data_buffer_dict[data_object.get_type()] = DataStructures.TimeStampedBuffer() - gy_value = data['mems']['gy']['gy'] - gy_ts = data['mems']['gy']['ts'] - gy_data = { - 'TIMESTAMP': gy_ts, - 'TIME': (gy_ts - self.__start_ts) / 1000000., - 'X': gy_value[0], - 'Y': gy_value[1], - 'Z': gy_value[2] - } + # store data object into the timestamped buffer dedicated to its type + ts_data_buffer_dict[data_object.get_type()][ts] = data_object - return gy_data + # start loading + with gzip.open(self.__segment_data_path) as f: - def __get_gp(self, data): + for item in f: + json.loads(item.decode('utf-8'), object_hook=decode) - gp_value = data['gp']['gp'] - gp_ts = data['gp']['ts'] - gp_data = { - 'TIMESTAMP': gp_ts, - 'TIME': (gp_ts - self.__start_ts) / 1000000., - 'X': gp_value[0], - 'Y': gp_value[1] - } + super().__init__(type(self).__name__, **ts_data_buffer_dict) - return gp_data + def keys(self): + """Get all registered data keys""" + return list(self.__dict__.keys())[2:-1] - def __get_pts(self, data): + def get_path(self): + return self.__segment_data_path - pts_value = data['pts']['pts'] - pts_ts = data['pts']['ts'] - pts_data = { - 'TIMESTAMP': pts_ts, - 'TIME': (pts_ts - self.__start_ts) / 1000000., - 'PTS': pts_value - } +class TobiiDataStream(threading.Thread): + """Capture Tobii Glasses Pro 2 data stream in separate thread.""" - return pts_data + def __init__(self, network_interface: TobiiNetworkInterface.TobiiNetworkInterface): + """Initialise thread super class as a deamon dedicated to data reception.""" - def run(self): - """Data reception function.""" + threading.Thread.__init__(self) + threading.Thread.daemon = True + + self.__network = network_interface + self.__data_socket = self.__network.make_socket() - while not self.stop_event.isSet(): + self.__data_queue = queue.Queue() - time.sleep(self.sleep) + self.__stop_event = threading.Event() + self.__read_lock = threading.Lock() - self.read_lock.acquire() + self.__sleep = 1. / self.__network.get_et_freq() - data = self.controller.get_data() + # prepare keep alive message + self.__keep_alive_msg = "{\"type\": \"live.data.unicast\", \"key\": \""+ str(uuid.uuid4()) +"\", \"op\": \"start\"}" + self.__keep_alive_thread = threading.Thread(target = self.__keep_alive) + self.__keep_alive_thread.daemon = True - # store only timestamped datas - if 'pts' in data: + def __del__(self): + """Stop data reception before destruction.""" - pts_data = data['pts'] + self.close() - if 'pts' in pts_data: + def __keep_alive(self): + """Maintain connection.""" - ac_ts = data['mems']['ac']['ts'] - gy_ts = data['mems']['gy']['ts'] - gp_ts = data['gp']['ts'] - pts_ts = pts_data['ts'] + while not self.__stop_event.isSet(): - # get start timestamp - if self.__start_ts == 0: + self.__network.send_keep_alive_msg(self.__data_socket, self.__keep_alive_msg) - # ignore -1 timestamp - valid_ts = [] - for ts in [ac_ts, gy_ts, gp_ts, pts_ts]: - if ts > 0: - valid_ts.append(ts) + time.sleep(1) - self.__start_ts = min(valid_ts) - #print(f'Tobii Data Frame: __start_ts = {self.__start_ts}') + def open(self): + """Start data reception.""" - #print(f'Tobii Data Frame: ac_ts = {ac_ts}, gy_ts = {gy_ts}, gp_ts = {gp_ts}, pts_ts = {pts_ts}') + self.__keep_alive_thread.start() + threading.Thread.start(self) - # ignore -1 timestamp and filter repetitions + def close(self): + """Stop data reception definitively.""" - if ac_ts != -1: - if len(self.__ac_buffer) == 0: - self.__ac_buffer.append(self.__get_ac(data)) - elif ac_ts != self.__ac_buffer[-1]['TIMESTAMP']: - self.__ac_buffer.append(self.__get_ac(data)) + self.__stop_event.set() - if gy_ts != -1: - if len(self.__gy_buffer) == 0: - self.__gy_buffer.append(self.__get_gy(data)) - elif gy_ts != self.__gy_buffer[-1]['TIMESTAMP']: - self.__gy_buffer.append(self.__get_gy(data)) + threading.Thread.join(self.__keep_alive_thread) + threading.Thread.join(self) - if gp_ts != -1: - if len(self.__gp_buffer) == 0: - self.__gp_buffer.append(self.__get_gp(data)) - elif gp_ts != self.__gp_buffer[-1]['TIMESTAMP']: - self.__gp_buffer.append(self.__get_gp(data)) + self.__data_socket.close() - if pts_ts != -1: - if len(self.__pts_buffer) == 0: - self.__pts_buffer.append(self.__get_pts(data)) - elif pts_ts != self.__pts_buffer[-1]['TIMESTAMP']: - self.__pts_buffer.append(self.__get_pts(data)) + def run(self): + """Store received data into a queue for further reading.""" - self.read_lock.release() + while not self.__stop_event.isSet(): - def read_accelerometer_data(self, timestamp: int = -1): - """Get accelerometer data at a given timestamp. - **Returns:** accelerometer dictionary - ``` - { - 'TIMESTAMP': int, - 'TIME': int, - 'X': float, - 'Y': float, - 'Z': float - } - ``` - """ + # wait + time.sleep(self.__sleep) - if len(self.__ac_buffer): + # lock data queue access + self.__read_lock.acquire() - self.read_lock.acquire() - - # TODO : find closest timestamp data - ac_data = self.__ac_buffer[-1].copy() + # write in data queue + data = self.__network.grab_data(self.__data_socket) + self.__data_queue.put(data) - self.read_lock.release() + # unlock data queue access + self.__read_lock.release() - return ac_data + def read(self): - else: + # create a dictionary of timestamped data buffers + ts_data_buffer_dict = DataStructures.DictObject('TobiiDataStream', **{}) - return {} + # if the data acquisition thread is not running + if self.__stop_event.isSet(): + return ts_data_buffer_dict - def read_accelerometer_buffer(self): - """Get accelerometer data buffer. - **Returns:** accelerometer dictionary array""" + # lock data queue access + self.__read_lock.acquire() - self.read_lock.acquire() - - ac_buffer = self.__ac_buffer.copy() + # read data queue + while not self.__data_queue.empty(): - self.read_lock.release() + data = self.__data_queue.get() - return ac_buffer + json_item = json.loads(data.decode('utf-8')) - def read_gyroscope_data(self, timestamp: int = -1): - """Get gyroscope data at a given timestamp. - **Returns:** gyroscope dictionary - ``` - { - 'TIMESTAMP': int, - 'TIME': int, - 'X': float, - 'Y': float, - 'Z': float - } - ``` - """ + # accept only valid data (e.g. with status value equal to 0) + if json_item.pop('s', -1) == 0: - if len(self.__gy_buffer): + # convert timestamp + ts = json_item.pop('ts') - self.read_lock.acquire() + #print(f'json_item at {ts}: {json_item}') + ''' + # keep first timestamp to offset all timestamps + if self.__ts_start == 0: + self.__ts_start = ts - # TODO : find closest timestamp data - gy_data = self.__gy_buffer[-1].copy() - - self.read_lock.release() - - return gy_data - - else: - - return {} - - def read_gyroscope_buffer(self): - """Get gyroscope data buffer. - **Returns:** gyroscope dictionary array""" - - self.read_lock.acquire() - - gy_buffer = self.__gy_buffer.copy() - - self.read_lock.release() - - return gy_buffer - - def read_gaze_data(self, timestamp: int = -1): - """Get gaze data at a given timestamp. - **Returns:** gaze dictionary - ``` - { - 'TIMESTAMP': int, - 'TIME': int, - 'X': float, - 'Y': float - } - ``` - """ - - if len(self.__gp_buffer): - - self.read_lock.acquire() + ts -= self.__ts_start - # TODO : find closest timestamp data - gp_data = self.__gp_buffer[-1].copy() - - self.read_lock.release() - - return gp_data - - else: - - return {} - - def read_gaze_buffer(self): - """Get gaze data buffer. - **Returns:** gaze dictionary array""" - - self.read_lock.acquire() - - gp_buffer = self.__gp_buffer.copy() - - self.read_lock.release() - - return gp_buffer - - def read_pts_data(self, timestamp: int = -1): - """Get Presentation Time Stamp (pts) data at a given timestamp. - **Returns:** pts dictionary - ``` - { - 'TIMESTAMP': int, - 'TIME': int, - 'PTS': int - } - ``` - """ - - if len(self.__pts_buffer): - - self.read_lock.acquire() + # ignore negative timestamp + if ts < 0: + break + ''' + # convert json data into data object + data_object_type = '_'.join(json_item.keys()) + data_object = DataStructures.DictObject(data_object_type, **json_item) + + # append a dedicated timestamped buffer for each data object type + if data_object.get_type() not in ts_data_buffer_dict.keys(): + ts_data_buffer_dict.append(data_object.get_type(), DataStructures.TimeStampedBuffer()) + + # store data object into the timestamped buffer dedicated to its type + ts_data_buffer_dict[data_object.get_type()][ts] = data_object - # TODO : find closest timestamp data - pts_data = self.__pts_buffer[-1].copy() + # unlock data queue access + self.__read_lock.release() - self.read_lock.release() - - return pts_data - - else: - - return {} - - def read_pts_buffer(self): - """Get Presentation Time Stamp (pts) data buffer. - **Returns:** pts dictionary array""" - - self.read_lock.acquire() - - pts_buffer = self.__pts_buffer.copy() - - self.read_lock.release() - - return pts_buffer - - def stop(self): - """Stop data reception definitively.""" - - self.stop_event.set() - threading.Thread.join(self) + return ts_data_buffer_dict diff --git a/src/argaze/TobiiGlassesPro2/TobiiEntities.py b/src/argaze/TobiiGlassesPro2/TobiiEntities.py index e3b0d9b..9dc41bc 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiEntities.py +++ b/src/argaze/TobiiGlassesPro2/TobiiEntities.py @@ -26,114 +26,6 @@ TOBII_SEGMENT_INFO_FILENAME = "segment.json" TOBII_SEGMENT_VIDEO_FILENAME = "fullstream.mp4" TOBII_SEGMENT_DATA_FILENAME = "livedata.json.gz" -class TobiiSegmentData(DataStructures.DictObject): - """Handle Tobii Glasses Pro 2 segment data file.""" - - def __init__(self, segment_data_path): - """Load segment data from segment directory then parse and register each recorded dataflow as a TimeStampedBuffer member of the TobiiSegmentData instance.""" - - self.__segment_data_path = segment_data_path - self.__ts_start = 0 - - ts_data_buffer_dict = {} - - # define a decoder function - def decode(json_item): - - # accept only valid data (e.g. with status value equal to 0) - if json_item.pop('s', -1) == 0: - - # convert timestamp - ts = json_item.pop('ts') - - # keep first timestamp to offset all timestamps - if self.__ts_start == 0: - self.__ts_start = ts - - ts -= self.__ts_start - - # ignore negative timestamp - if ts < 0: - return - - # convert json data into data object - data_object_type = '_'.join(json_item.keys()) - data_object = DataStructures.DictObject(data_object_type, **json_item) - - # append a dedicated timestamped buffer for each data object type - if data_object.get_type() not in ts_data_buffer_dict.keys(): - ts_data_buffer_dict[data_object.get_type()] = DataStructures.TimeStampedBuffer() - - # store data object into the timestamped buffer dedicated to its type - ts_data_buffer_dict[data_object.get_type()][ts] = data_object - - # start loading - with gzip.open(self.__segment_data_path) as f: - - for item in f: - json.loads(item.decode('utf-8'), object_hook=decode) - - super().__init__(type(self).__name__, **ts_data_buffer_dict) - - def keys(self): - """Get all registered data keys""" - return list(self.__dict__.keys())[2:-1] - - def get_path(self): - return self.__segment_data_path - -class TobiiVideoFrame(DataStructures.DictObject): - """Define tobii video frame""" - - def __init__(self, matrix, width, height, pts): - - super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts}) - -class TobiiSegmentVideo(): - """Handle Tobii Glasses Pro 2 segment video file.""" - - def __init__(self, segment_video_path): - """Load segment video from segment directory""" - - self.__segment_video_path = segment_video_path - self.__container = av.open(self.__segment_video_path) - self.__stream = self.__container.streams.video[0] - - self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH)) - self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT)) - - def get_path(self): - return self.__segment_video_path - - def get_duration(self): - return float(self.__stream.duration * self.__stream.time_base) - - def get_frame_number(self): - return self.__stream.frames - - def get_width(self): - return self.__width - - def get_height(self): - return self.__height - - def frames(self): - return self.__iter__() - - def __iter__(self): - - # start decoding - self.__container.decode(self.__stream) - - return self - - def __next__(self): - - frame = self.__container.decode(self.__stream).__next__() - - # return micro second timestamp and frame data - return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) - class TobiiSegment: """Handle Tobii Glasses Pro 2 segment info.""" diff --git a/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py b/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py new file mode 100644 index 0000000..13c0b9a --- /dev/null +++ b/src/argaze/TobiiGlassesPro2/TobiiNetworkInterface.py @@ -0,0 +1,224 @@ +import logging +import sys +import socket +import threading +import json +import time + +# python2 backwards compatibility for errors +if sys.version_info[0] < 3: + class ConnectionError(BaseException): + pass + +try: + import netifaces + TOBII_DISCOVERY_ALLOWED = True +except: + TOBII_DISCOVERY_ALLOWED = False + +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request + from urllib.error import URLError, HTTPError + +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, HTTPError, URLError + +socket.IPPROTO_IPV6 = 41 + +class TobiiNetworkInterface(): + """Handle network connection to Tobii glasses Pro 2 device. + It is a major rewrite of [tobiiglassesctrl/controller.py](https://github.com/ddetommaso/TobiiGlassesPyController/blob/master/tobiiglassesctrl/controller.py).""" + + def __init__(self, address = None): + + self.udpport = 49152 + self.address = address + self.iface_name = None + + if self.address is None: + + data, address = self.__discover_device() + + if address is None: + raise ConnectionError("No device found using discovery process") + else: + try: + self.address = data["ipv4"] + except: + self.address = address + + if "%" in self.address: + if sys.platform == "win32": + self.address,self.iface_name = self.address.split("%") + else: + self.iface_name = self.address.split("%")[1] + + if ':' in self.address: + self.base_url = 'http://[%s]' % self.address + else: + self.base_url = 'http://' + self.address + + self.__peer = (self.address, self.udpport) + + def make_socket(self): + + iptype = socket.AF_INET + + if ':' in self.__peer[0]: + iptype = socket.AF_INET6 + + res = socket.getaddrinfo(self.__peer[0], self.__peer[1], socket.AF_UNSPEC, socket.SOCK_DGRAM, 0, socket.AI_PASSIVE) + family, socktype, proto, canonname, sockaddr = res[0] + new_socket = socket.socket(family, socktype, proto) + + new_socket.settimeout(5.0) + + try: + if iptype == socket.AF_INET6: + new_socket.setsockopt(socket.SOL_SOCKET, 25, 1) + + except socket.error as e: + if e.errno == 1: + logging.warning("Binding to a network interface is permitted only for root users.") + + return new_socket + + def connect(self, timeout = None): + + return self.wait_for_status('/api/system/status', 'sys_status', ['ok']) == 'ok' + + def __discover_device(self): + + if TOBII_DISCOVERY_ALLOWED == False: + logging.error("Device discovery is not available due to a missing dependency (netifaces)") + exit(1) + + logging.debug("Looking for a Tobii Pro Glasses 2 device ...") + + MULTICAST_ADDR = 'ff02::1' + PORT = 13006 + + for i in netifaces.interfaces(): + + if netifaces.AF_INET6 in netifaces.ifaddresses(i).keys(): + + if "%" in netifaces.ifaddresses(i)[netifaces.AF_INET6][0]['addr']: + + if_name = netifaces.ifaddresses(i)[netifaces.AF_INET6][0]['addr'].split("%")[1] + if_idx = socket.getaddrinfo(MULTICAST_ADDR + "%" + if_name, PORT, socket.AF_INET6, socket.SOCK_DGRAM)[0][4][3] + + s6 = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s6.settimeout(30.0) + s6.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, if_idx) + s6.bind(('::', PORT)) + + PORT_OUT = PORT if sys.platform == 'win32' or sys.platform == 'darwin' else PORT + 1 + + try: + discover_json = '{"type":"discover"}' + s6.sendto(discover_json.encode('utf-8'), (MULTICAST_ADDR, PORT_OUT)) + + logging.debug("Discover request sent to %s on interface %s " % ( str((MULTICAST_ADDR, PORT_OUT)),if_name) ) + logging.debug("Waiting for a reponse from the device ...") + + data, address = s6.recvfrom(1024) + jdata = json.loads(data.decode('utf-8')) + + logging.debug("From: " + address[0] + " " + str(data)) + logging.debug("Tobii Pro Glasses found with address: [%s]" % address[0]) + + addr = address[0] + + if sys.version_info.major == 3 and sys.version_info.minor >= 8: + addr = address[0] + '%' + if_name + + return (jdata, addr) + + except: + logging.debug("No device found on interface %s" % if_name) + + logging.debug("The discovery process did not find any device!") + + return (None, None) + + def get_request(self, api_action): + + url = self.base_url + api_action + res = urlopen(url).read() + try: + data = json.loads(res.decode('utf-8')) + except json.JSONDecodeError: + data = None + return data + + def post_request(self, api_action, data=None, wait_for_response=True): + + url = self.base_url + api_action + req = Request(url) + req.add_header('Content-Type', 'application/json') + data = json.dumps(data) + + logging.debug("Sending JSON: " + str(data)) + + if wait_for_response is False: + threading.Thread(target=urlopen, args=(req, data.encode('utf-8'),)).start() + return None + + response = urlopen(req, data.encode('utf-8')) + res = response.read() + + logging.debug("Response: " + str(res)) + + try: + res = json.loads(res.decode('utf-8')) + + except: + pass + + return res + + def send_keep_alive_msg(self, socket, msg): + + res = socket.sendto(msg.encode('utf-8'), self.__peer) + + def grab_data(self, socket): + + try: + data, address = socket.recvfrom(1024) + return data + + except TimeoutError: + + logging.error("A timeout occurred while receiving data") + + def wait_for_status(self, api_action, key, values, timeout = None): + + url = self.base_url + api_action + running = True + + while running: + + req = Request(url) + req.add_header('Content-Type', 'application/json') + + try: + + response = urlopen(req, None, timeout = timeout) + + except URLError as e: + + logging.error(e.reason) + return -1 + + data = response.read() + json_data = json.loads(data.decode('utf-8')) + + if json_data[key] in values: + running = False + + time.sleep(1) + + return json_data[key] diff --git a/src/argaze/TobiiGlassesPro2/TobiiVideo.py b/src/argaze/TobiiGlassesPro2/TobiiVideo.py index b927b7f..748967a 100644 --- a/src/argaze/TobiiGlassesPro2/TobiiVideo.py +++ b/src/argaze/TobiiGlassesPro2/TobiiVideo.py @@ -1,96 +1,156 @@ #!/usr/bin/env python import threading +import uuid +import time +import copy + +from argaze import DataStructures +from argaze.TobiiGlassesPro2 import TobiiNetworkInterface import av import numpy -class TobiiVideoThread(threading.Thread): - """Handle video camera stream capture in a separate thread.""" +class TobiiVideoFrame(DataStructures.DictObject): + """Define tobii video frame""" + + def __init__(self, matrix, width, height, pts): + + super().__init__(type(self).__name__, **{'matrix': matrix, 'width': width, 'height': height, 'pts': pts}) + +class TobiiVideoSegment(): + """Handle Tobii Glasses Pro 2 segment video file.""" + + def __init__(self, segment_video_path): + """Load segment video from segment directory""" + + self.__segment_video_path = segment_video_path + self.__container = av.open(self.__segment_video_path) + self.__stream = self.__container.streams.video[0] + + self.__width = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_WIDTH)) + self.__height = int(cv.VideoCapture(self.__segment_video_path).get(cv.CAP_PROP_FRAME_HEIGHT)) + + def get_path(self): + return self.__segment_video_path + + def get_duration(self): + return float(self.__stream.duration * self.__stream.time_base) + + def get_frame_number(self): + return self.__stream.frames + + def get_width(self): + return self.__width + + def get_height(self): + return self.__height + + def frames(self): + return self.__iter__() + + def __iter__(self): + + # start decoding + self.__container.decode(self.__stream) + + return self + + def __next__(self): - def __init__(self, controller): - """Initialise thread super class and prepare camera video stream reception.""" + frame = self.__container.decode(self.__stream).__next__() + + # return micro second timestamp and frame data + return frame.time * 1000000, TobiiVideoFrame(frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) + +class TobiiVideoStream(threading.Thread): + """Capture Tobii Glasses Pro 2 video camera stream.""" + + def __init__(self, network_interface: TobiiNetworkInterface.TobiiNetworkInterface): + """Initialise video stream reception.""" threading.Thread.__init__(self) - self.stop_event = threading.Event() - self.read_lock = threading.Lock() + threading.Thread.daemon = True - self.controller = controller + self.__network = network_interface + self.__video_socket = self.__network.make_socket() - self.fps = self.controller.get_video_freq() + self.__stop_event = threading.Event() + self.__read_lock = threading.Lock() - self.read_lock.acquire() + self.__frame_tuple = None - self.__frame = numpy.zeros((1, 1, 3), numpy.uint8) - self.__width = 0 - self.__height = 0 - self.__pts_buffer = [] + self.__sleep = 1. / self.__network.get_video_freq() - self.read_lock.release() + # prepare keep alive message + self.__keep_alive_msg = "{\"type\": \"live.video.unicast\",\"key\": \""+ str(uuid.uuid4()) +"_video\", \"op\": \"start\"}" + self.__keep_alive_thread = threading.Timer(0, self.__keep_alive) + self.__keep_alive_thread.daemon = True def __del__(self): - pass + """Stop data reception before destruction.""" - def run(self): - """Video camera stream capture function.""" + self.close() - # start Tobii glasses stream capture - self.__container = av.open(f'rtsp://{self.controller.get_address()}:8554/live/scene', options={'rtsp_transport': 'tcp'}) - self.__stream = self.__container.streams.video[0] - - for f in self.__container.decode(self.__stream): + def __keep_alive(self): + """Maintain connection.""" - if self.stop_event.isSet(): - break + while not self.__stop_event.isSet(): + + self.__network.send_keep_alive_msg(self.__video_socket, self.__keep_alive_msg) + + time.sleep(1) - self.read_lock.acquire() - - self.__frame = f.to_ndarray(format='bgr24') - self.__width = f.width - self.__height = f.height - self.__pts_buffer.append({'TIME':f.time, 'PTS': f.pts}) - - self.read_lock.release() + def open(self): + """Start data reception.""" - def read(self) : - """Read video frame. - **Returns:** frame, frame width, frame height, frame time, frame pts.""" + self.__keep_alive_thread.start() + threading.Thread.start(self) - # if stopped, return blank frame - if self.stop_event.isSet(): - return numpy.zeros((1, 1, 3), numpy.uint8) + def close(self): + """Stop data reception definitively.""" - # else - self.read_lock.acquire() + self.__stop_event.set() - frame_copy = self.__frame.copy() - width_copy = self.__width - height_copy = self.__height + threading.Thread.join(self.__keep_alive_thread) + threading.Thread.join(self) - if len(self.__pts_buffer): - time_copy = self.__pts_buffer[-1]['TIME'] - pts_copy = self.__pts_buffer[-1]['PTS'] - else: - time_copy = -1 - pts_copy = -1 + self.__video_socket.close() - self.read_lock.release() + def run(self): + """Store frame for further reading.""" - return frame_copy, width_copy, height_copy, time_copy, pts_copy + container = av.open(f'rtsp://{self.__network.get_address()}:8554/live/scene', options={'rtsp_transport': 'tcp'}) + stream = container.streams.video[0] - def read_pts_buffer(self): - """Get Presentation Time Stamp data buffer.""" + for frame in container.decode(stream): - self.read_lock.acquire() - - pts_buffer = self.__pts_buffer.copy() + # quit if the video acquisition thread have been stopped + if self.__stop_event.isSet(): + break - self.read_lock.release() + # lock frame access + self.__read_lock.acquire() - return pts_buffer + # store frame time, matrix, width, height and pts into a tuple + self.__frame_tuple = (frame.time, frame.to_ndarray(format='bgr24'), frame.width, frame.height, frame.pts) - def stop(self): - """Stop video camera stream capture definitively.""" + # unlock frame access + self.__read_lock.release() - self.stop_event.set() - threading.Thread.join(self) + def read(self): + + # if the video acquisition thread have been stopped or isn't started + if self.__stop_event.isSet() or self.__frame_tuple == None: + return -1, TobiiVideoFrame(numpy.zeros((1, 1, 3), numpy.uint8), 1, 1, -1) + + # lock frame access + self.__read_lock.acquire() + + # copy frame tuple + frame_tuple = copy.deepcopy(self.__frame_tuple) + + # unlock frame access + self.__read_lock.release() + + return frame_tuple[0] * 1000000, TobiiVideoFrame(frame_tuple[1], frame_tuple[2], frame_tuple[3], frame_tuple[4]) diff --git a/src/argaze/TobiiGlassesPro2/__init__.py b/src/argaze/TobiiGlassesPro2/__init__.py index 3c92c1f..329402c 100644 --- a/src/argaze/TobiiGlassesPro2/__init__.py +++ b/src/argaze/TobiiGlassesPro2/__init__.py @@ -2,4 +2,4 @@ .. include:: README.md """ __docformat__ = "restructuredtext" -__all__ = ['TobiiEntities','TobiiController', 'TobiiData', 'TobiiVideo']
\ No newline at end of file +__all__ = ['TobiiEntities', 'TobiiNetworkInterface', 'TobiiController', 'TobiiData', 'TobiiVideo']
\ No newline at end of file diff --git a/src/argaze/utils/README.md b/src/argaze/utils/README.md index 461d82d..ffe2c16 100644 --- a/src/argaze/utils/README.md +++ b/src/argaze/utils/README.md @@ -30,10 +30,10 @@ python ./src/argaze/utils/export_calibration_board.py 7 5 5 3 -o export python ./src/argaze/utils/calibrate_tobii_camera.py 7 5 5 3 -t IP_ADDRESS -o export/tobii_camera.json ``` -- Display Tobii Glasses Pro 2 gaze and camera video stream (replace IP_ADDRESS) +- Display Tobii Glasses Pro 2 camera video stream (replace IP_ADDRESS) and gaze ``` -python ./src/argaze/utils/display_tobii_gaze.py -t IP_ADDRESS +python ./src/argaze/utils/live_tobii_session.py -t IP_ADDRESS ``` - Record a Tobii Glasses Pro 2 'Test' session for a participant '1' on Tobii interface's SD card (replace IP_ADDRESS). diff --git a/src/argaze/utils/calibrate_tobii_camera.py b/src/argaze/utils/calibrate_tobii_camera.py index 93751ba..0381f75 100644 --- a/src/argaze/utils/calibrate_tobii_camera.py +++ b/src/argaze/utils/calibrate_tobii_camera.py @@ -26,30 +26,29 @@ def main(): parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) parser.add_argument('columns', metavar='COLS_NUMBER', type=int, default=7, help='number of columns') parser.add_argument('rows', metavar='ROWS_NUMBER', type=int, default=5, help='number of rows') - parser.add_argument('square_size', metavar='SQUARE_SIZE', type=int, default=5, help='square size (cm)') - parser.add_argument('marker_size', metavar='MARKER_SIZE', type=int, default=5, help='marker size (cm)') + parser.add_argument('square_size', metavar='SQUARE_SIZE', type=float, default=5, help='square size (cm)') + parser.add_argument('marker_size', metavar='MARKER_SIZE', type=float, default=3, help='marker size (cm)') parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip') parser.add_argument('-o', '--output', metavar='OUT', type=str, default='.', help='destination filepath') parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_4X4_50', help='aruco marker dictionnary') args = parser.parse_args() - # create tobii controller + # Create tobii controller tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1) - # create tobii video thread - tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller) - tobii_video_thread.start() + # Enable tobii video stream + tobii_video_stream = tobii_controller.enable_video_stream() - # create aruco camera + # Create aruco camera aruco_camera = ArUcoCamera.ArUcoCamera() - # create aruco board + # Create aruco board aruco_board = ArUcoBoard.ArUcoBoard(args.dictionary, args.columns, args.rows, args.square_size, args.marker_size) - # create aruco tracker + # Create aruco tracker aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, args.marker_size, aruco_camera) - # start tobii glasses streaming + # Start tobii glasses streaming tobii_controller.start_streaming() print("Camera calibration starts") @@ -64,32 +63,35 @@ def main(): # capture loop try: - # wait 1ms between each frame until 'Esc' key is pressed - while cv.waitKey(1) != 27: + while tobii_video_stream.is_alive(): # capture frame with a full displayed board - frame, frame_width, frame_height, frame_time, frame_pts = tobii_video_thread.read() + video_ts, video_frame = tobii_video_stream.read() # track all markers in the board - aruco_tracker.track_board(frame, aruco_board, expected_markers_number) + aruco_tracker.track_board(video_frame.matrix, aruco_board, expected_markers_number) # draw only markers - aruco_tracker.draw(frame) + aruco_tracker.draw(video_frame.matrix) # draw current calibration data count - cv.putText(frame, f'Capture: {aruco_camera.get_calibration_data_count()}', (50, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv.LINE_AA) - cv.imshow('Tobii Camera Calibration', frame) + cv.putText(video_frame.matrix, f'Capture: {aruco_camera.get_calibration_data_count()}', (50, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv.LINE_AA) + cv.imshow('Tobii Camera Calibration', video_frame.matrix) # if all board corners are detected if aruco_tracker.get_board_corners_number() == expected_corners_number: # draw board corners to notify a capture is done - aruco_tracker.draw_board(frame) + aruco_tracker.draw_board(video_frame.matrix) # append data aruco_camera.store_calibration_data(aruco_tracker.get_board_corners(), aruco_tracker.get_board_corners_ids()) - cv.imshow('Tobii Camera Calibration', frame) + cv.imshow('Tobii Camera Calibration', video_frame.matrix) + + # close window using 'Esc' key + if cv.waitKey(1) == 27: + break # exit on 'ctrl+C' interruption except KeyboardInterrupt: @@ -98,14 +100,11 @@ def main(): # stop frame display cv.destroyAllWindows() - # stop tobii objects - tobii_video_thread.stop() - + # Stop tobii glasses streaming tobii_controller.stop_streaming() - tobii_controller.close() print('\nCalibrating camera...') - aruco_camera.calibrate(aruco_board, frame_width, frame_height) + aruco_camera.calibrate(aruco_board, video_frame.width, video_frame.height) print('\nCalibration succeeded!') print(f'\nRMS:\n{aruco_camera.get_rms()}') diff --git a/src/argaze/utils/display_tobii_gaze.py b/src/argaze/utils/display_tobii_gaze.py deleted file mode 100644 index 18b89f8..0000000 --- a/src/argaze/utils/display_tobii_gaze.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -import argparse -import os, time - -from argaze.TobiiGlassesPro2 import * - -import cv2 as cv -import numpy - -def main(): - """ - Capture video camera and display gaze point - """ - - # manage arguments - parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) - parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip') - - args = parser.parse_args() - - # create tobii controller - tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1) - - # calibrate tobii glasses - tobii_controller.calibrate() - - # create tobii data thread - tobii_data_thread = TobiiData.TobiiDataThread(tobii_controller) - tobii_data_thread.start() - - # create tobii video thread - tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller) - tobii_video_thread.start() - - # start tobii glasses streaming - tobii_controller.start_streaming() - - # display loop - try: - - # wait 1ms between each frame until 'Esc' key is pressed - while cv.waitKey(1) != 27: - - frame, frame_width, frame_height, frame_time, pts = tobii_video_thread.read() - - # draw tobii gaze - # TODO : sync gaze data according frame pts - gp_data = tobii_data_thread.read_gaze_data(pts) - if 'TIMESTAMP' in gp_data: - pointer = (int(gp_data['X'] * frame_width), int(gp_data['Y'] * frame_height)) - cv.circle(frame, pointer, 4, (0, 255, 255), -1) - else: - pointer = (0, 0) - - # display frame - cv.imshow('Tobii Camera Record', frame) - - # exit on 'ctrl+C' interruption - except KeyboardInterrupt: - pass - - # stop frame display - cv.destroyAllWindows() - - # stop tobii objects - tobii_video_thread.stop() - tobii_data_thread.stop() - - tobii_controller.stop_streaming() - tobii_controller.close() - -if __name__ == '__main__': - - main()
\ No newline at end of file diff --git a/src/argaze/utils/export_tobii_segment_aruco_markers.py b/src/argaze/utils/export_tobii_segment_aruco_markers.py new file mode 100644 index 0000000..11c5c1b --- /dev/null +++ b/src/argaze/utils/export_tobii_segment_aruco_markers.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +import argparse +import bisect + +from argaze import GazeFeatures +from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo +from argaze.ArUcoMarkers import * +from argaze.RegionOfInterest import * + +import numpy + +import cv2 as cv + +def main(): + """ + Replay Tobii segment video + """ + + # manage arguments + parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) + parser.add_argument('-s', '--segment_path', metavar='SEGMENT_PATH', type=str, default=None, help='segment path') + args = parser.parse_args() + + if args.segment_path != None: + + # Load a tobii segment + tobii_segment = TobiiEntities.TobiiSegment(args.segment_path) + + # create aruco camera + aruco_camera = ArUcoCamera.ArUcoCamera() + aruco_camera.load_calibration_file('/Users/Robotron/Developpements/ArGaze/export/tobii_camera.json') + + # create aruco tracker + aruco_tracker = ArUcoTracker.ArUcoTracker('DICT_ARUCO_ORIGINAL', 7.5, aruco_camera) # aruco dictionary, marker length (cm), camera + + # Load a tobii segment video + tobii_segment_video = tobii_segment.load_video() + print(f'Video duration: {tobii_segment_video.get_duration()}, frame number: {tobii_segment_video.get_frame_number()}, width: {tobii_segment_video.get_width()}, height: {tobii_segment_video.get_height()}') + + # create ROIs 3D scene + roi3D_scene = ROI3DScene.ROI3DScene() + roi3D_scene.load('/Users/Robotron/Developpements/ArGaze/export/test.obj') + + # replay loop + try: + + last_ts = 0 + for frame_ts, frame in tobii_segment_video.frames(): + + if frame_ts > last_ts: + + delay = int((frame_ts - last_ts) / 1000) + + if cv.waitKey(delay) == 27: + break + + # track markers with pose estimation and draw them + aruco_tracker.track(frame.matrix) + aruco_tracker.draw(frame.matrix) + + # project 3D scenes related to each aruco markers + if aruco_tracker.get_markers_number(): + + for (i, marker_id) in enumerate(aruco_tracker.get_markers_ids()): + + # TODO : select different 3D scenes depending on aruco id + + marker_rotation = aruco_tracker.get_marker_rotation(i) + marker_translation = aruco_tracker.get_marker_translation(i) + + roi3D_scene.set_rotation(marker_rotation) + roi3D_scene.set_translation(marker_translation) + + # zero distorsion matrix + D0 = numpy.asarray([0.0, 0.0, 0.0, 0.0, 0.0]) + + # DON'T APPLY CAMERA DISTORSION : it projects points which are far from the frame into it + # This hack isn't realistic but as the gaze will mainly focus on centered ROI, where the distorsion is low, it is acceptable. + roi2D_scene = roi3D_scene.project(aruco_camera.get_K(), D0) + + # check if gaze is inside 2D rois + #roi2D_scene.inside(pointer) + + # draw 2D rois + roi2D_scene.draw(frame.matrix) + + cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix) + last_ts = frame_ts + + # exit on 'ctrl+C' interruption + except KeyboardInterrupt: + pass + + # stop frame display + cv.destroyAllWindows() + +if __name__ == '__main__': + + main()
\ No newline at end of file diff --git a/src/argaze/utils/track_aruco_rois_with_tobii_glasses.py b/src/argaze/utils/live_tobii_aruco_detection.py index 334a6bb..82837d6 100644 --- a/src/argaze/utils/track_aruco_rois_with_tobii_glasses.py +++ b/src/argaze/utils/live_tobii_aruco_detection.py @@ -3,6 +3,7 @@ import argparse import os +from argaze.TobiiGlassesPro2 import * from argaze.ArUcoMarkers import ArUcoTracker, ArUcoCamera from argaze.RegionOfInterest import * from argaze.TobiiGlassesPro2 import * @@ -20,45 +21,43 @@ def main(): Export all collected datas into an export folder for further analysis. """ - # manage arguments + # Manage arguments parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.10', help='tobii glasses ip') parser.add_argument('-c', '--camera_calibration', metavar='CAM_CALIB', type=str, default='tobii_camera.json', help='json camera calibration filepath') parser.add_argument('-s', '--roi_scene', metavar='ROI_SCENE', type=str, default='roi3D_scene.obj', help='obj roi scene filepath') parser.add_argument('-o', '--output', metavar='OUT', type=str, default='.', help='destination path') - parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_4X4_50', help='aruco marker dictionnary') + parser.add_argument('-d', '--dictionary', metavar='DICT', type=str, default='DICT_ARUCO_ORIGINAL', help='aruco marker dictionnary') parser.add_argument('-m', '--marker_size', metavar='MKR', type=int, default=6, help='aruco marker size (cm)') args = parser.parse_args() - # create tobii controller + # Create tobii controller tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1) - # calibrate tobii glasses - tobii_controller.calibrate() + # Calibrate tobii glasses + #tobii_controller.calibrate() - # create tobii data thread - tobii_data_thread = TobiiData.TobiiDataThread(tobii_controller) - tobii_data_thread.start() + # Create tobii data stream + #tobii_data_stream = TobiiData.TobiiDataStream(tobii_controller) - # create tobii video thread - tobii_video_thread = TobiiVideo.TobiiVideoThread(tobii_controller) - tobii_video_thread.start() + # Create tobii video stream + tobii_video_stream = TobiiVideo.TobiiVideoStream(tobii_controller) # create aruco camera aruco_camera = ArUcoCamera.ArUcoCamera() aruco_camera.load_calibration_file(args.camera_calibration) - # create aruco tracker - aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, 6, aruco_camera) # aruco dictionaries, marker length (cm), camera + # Create aruco tracker + aruco_tracker = ArUcoTracker.ArUcoTracker(args.dictionary, 7.5, aruco_camera) # aruco dictionary, marker length (cm), camera - # create ROIs 3D scene + # Create ROIs 3D scene roi3D_scene = ROI3DScene.ROI3DScene() roi3D_scene.load(args.roi_scene) - # start tobii glasses streaming + # Start tobii glasses streaming tobii_controller.start_streaming() - # process video frames + # Process video frames frame_time = 0 last_frame_time = 0 roi2D_buffer = [] @@ -67,14 +66,15 @@ def main(): # tracking loop try: - # wait 1ms between each frame until 'Esc' key is pressed - while cv.waitKey(1) != 27: - - frame, frame_width, frame_height, frame_time, pts = tobii_video_thread.read() + for frame_ts, frame in tobii_video_stream.frames(): + # close window using 'Esc' key + if cv.waitKey(1) == 27: + break + # draw tobii gaze # TODO : sync gaze data according frame pts - gp_data = tobii_data_thread.read_gaze_data(pts) + gp_data = tobii_data_stream.read_gaze_data(pts) if 'TIMESTAMP' in gp_data: pointer = (int(gp_data['X'] * frame_width), int(gp_data['Y'] * frame_height)) cv.circle(frame, pointer, 4, (0, 255, 255), -1) @@ -137,12 +137,8 @@ def main(): cv.destroyAllWindows() last_frame_time = frame_time - # stop tobii objects - tobii_video_thread.stop() - tobii_data_thread.stop() - + # stop tobii glasses streaming tobii_controller.stop_streaming() - tobii_controller.close() # create a pandas DataFrame for each buffer ac_dataframe = pandas.DataFrame(tobii_data_thread.read_accelerometer_buffer(), columns=['TIMESTAMP', 'TIME', 'X', 'Y', 'Z']) diff --git a/src/argaze/utils/live_tobii_session.py b/src/argaze/utils/live_tobii_session.py new file mode 100644 index 0000000..0181e23 --- /dev/null +++ b/src/argaze/utils/live_tobii_session.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import argparse +import os, time + +from argaze import GazeFeatures +from argaze.TobiiGlassesPro2 import * + +import cv2 as cv +import numpy + +def main(): + """ + Capture video camera and display gaze point + """ + + # Manage arguments + parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) + parser.add_argument('-t', '--tobii_ip', metavar='TOBII_IP', type=str, default='192.168.1.12', help='tobii glasses ip') + + args = parser.parse_args() + + # Create tobii controller + tobii_controller = TobiiController.TobiiController(args.tobii_ip, 'ArGaze', 1) + + # Calibrate tobii glasses + tobii_controller.calibrate() + + # Enable tobii data stream + tobii_data_stream = tobii_controller.enable_data_stream() + + # Enable tobii video stream + tobii_video_stream = tobii_controller.enable_video_stream() + + # Start streaming + tobii_controller.start_streaming() + + # Live video stream capture loop + try: + + while tobii_video_stream.is_alive(): + + video_ts, video_frame = tobii_video_stream.read() + + try: + + # get last gaze position + last_ts, last_gaze_position = tobii_data_stream.read().gidx_l_gp.pop_last() + + # Draw tobii gaze pointer + pointer = (int(last_gaze_position.gp[0] * video_frame.width), int(last_gaze_position.gp[1] * video_frame.height)) + cv.circle(video_frame.matrix, pointer, 4, (0, 255, 255), -1) + + # when gidx_l_gp key not received during last frame + except (KeyError, AttributeError): + pass + + # close window using 'Esc' key + if cv.waitKey(1) == 27: + break + + cv.imshow(f'Live Tobii Camera', video_frame.matrix) + + # exit on 'ctrl+C' interruption + except KeyboardInterrupt: + pass + + # Stop frame display + cv.destroyAllWindows() + + # Stop streaming + tobii_controller.stop_streaming() + +if __name__ == '__main__': + + main()
\ No newline at end of file diff --git a/src/argaze/utils/replay_tobii_session.py b/src/argaze/utils/replay_tobii_session.py new file mode 100644 index 0000000..b911d09 --- /dev/null +++ b/src/argaze/utils/replay_tobii_session.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +import argparse +import bisect + +from argaze import GazeFeatures +from argaze.TobiiGlassesPro2 import TobiiEntities, TobiiVideo, TobiiData + +import numpy + +import cv2 as cv + +def main(): + """ + Replay Tobii segment video + """ + + # manage arguments + parser = argparse.ArgumentParser(description=main.__doc__.split('-')[0]) + parser.add_argument('-s', '--segment_path', metavar='SEGMENT_PATH', type=str, default=None, help='segment path') + args = parser.parse_args() + + if args.segment_path != None: + + # Load a tobii segment + tobii_segment = TobiiEntities.TobiiSegment(args.segment_path) + + # Load a tobii segment video + tobii_segment_video = tobii_segment.load_video() + print(f'Video duration: {tobii_segment_video.get_duration()}, frame number: {tobii_segment_video.get_frame_number()}, width: {tobii_segment_video.get_width()}, height: {tobii_segment_video.get_height()}') + + # Load a tobii segment data + tobii_segment_data = tobii_segment.load_data() + print(f'Data keys: {tobii_segment_data.keys()}') + + # Access to timestamped gaze position data buffer + tobii_ts_gaze_positions = tobii_segment_data.gidx_l_gp + + print(f'{len(tobii_ts_gaze_positions)} gaze positions loaded') + + # video and data replay loop + try: + + for frame_ts, frame in tobii_segment_video.frames(): + + # close window using 'Esc' key + if cv.waitKey(1) == 27: + break + + cv.imshow(f'Segment {tobii_segment.get_id()} video', frame.matrix) + + # exit on 'ctrl+C' interruption + except KeyboardInterrupt: + pass + + # stop frame display + cv.destroyAllWindows() + +if __name__ == '__main__': + + main()
\ No newline at end of file diff --git a/src/argaze/utils/synchronise_timestamped_data.py b/src/argaze/utils/synchronise_timestamped_data.py new file mode 100644 index 0000000..0c22072 --- /dev/null +++ b/src/argaze/utils/synchronise_timestamped_data.py @@ -0,0 +1,50 @@ + + + + + + + + # Synchronise video and gaze + vts_ts, vts = tobii_vts.pop_first() + vts_offset = (vts_ts - vts.vts) / 1000 + + print(f'Init >>> vts_offset = {vts_offset}') + + last_gaze_position_index = -1 + closest_gaze_position_ts, closest_gaze_position = None, None + + last_fixation_index = -1 + closest_fixation_ts, closest_fixation = None, None + + for frame_ts, frame in tobii_segment_video.frames(): + + frame_ts = frame_ts / 1000 + + if frame_ts > vts.vts / 1000: + if len(tobii_vts) > 0: + vts_ts, vts = tobii_vts.pop_first() + vts_offset = (vts_ts - vts.vts) / 1000 + print(f'{frame_ts / 1000} >>> New vts_offset = {vts_offset / 1000}') + + # Find closest gaze position + closest_index = bisect.bisect_left(list(generic_ts_gaze_positions.keys()), frame_ts + vts_offset) - 1 + if closest_index > last_gaze_position_index: + + # pop data until closest_index + while last_gaze_position_index <= closest_index: + closest_gaze_position_ts, closest_gaze_position = generic_ts_gaze_positions.pop_first() + last_gaze_position_index += 1 + + print(f'{frame_ts / 1000} *** New closest gaze position: index = {closest_index}, ts = {closest_gaze_position_ts / 1000}, {closest_gaze_position}') + + # Find closest fixation + closest_index = bisect.bisect_left(list(fixation_analyser.fixations.keys()), frame_ts + vts_offset) - 1 + if closest_index > last_fixation_index: + + # pop data until closest_index + while last_fixation_index <= closest_index: + closest_ts, closest_fixation = fixation_analyser.fixations.pop_first() + last_fixation_index += 1 + + print(f'{frame_ts / 1000} /// New closest fixation: index= {closest_index}, ts = {closest_ts / 1000}, {closest_fixation}')
\ No newline at end of file |