From 8da7f2e1e0215df52903c14fb30203624bfcf1ee Mon Sep 17 00:00:00 2001 From: Théo de la Hogue Date: Wed, 20 Sep 2023 14:07:13 +0200 Subject: Improving aruco markers gaze analysis pipeline documentation. --- docs/img/aruco_camera_aoi_projection.png | Bin 0 -> 57019 bytes docs/img/aruco_camera_frame.png | Bin 54853 -> 49201 bytes .../aruco_camera_gaze_movement_identification.png | Bin 0 -> 56059 bytes docs/img/aruco_camera_markers_detection.png | Bin 51729 -> 52844 bytes docs/img/aruco_camera_pose_estimation.png | Bin 0 -> 51798 bytes .../aruco_markers_pipeline/aoi_analysis.md | 58 +++++ .../aruco_markers_pipeline/aoi_description.md | 4 +- .../aruco_markers_pipeline/aoi_projection.md | 159 +++++++++++++ .../aruco_camera_configuration_and_execution.md | 115 ---------- .../aruco_markers_pipeline/aruco_scene.md | 245 --------------------- .../configuration_and_execution.md | 138 ++++++++++++ .../aruco_markers_pipeline/introduction.md | 7 +- .../aruco_markers_pipeline/pose_estimation.md | 82 +++++++ 13 files changed, 443 insertions(+), 365 deletions(-) create mode 100644 docs/img/aruco_camera_aoi_projection.png create mode 100644 docs/img/aruco_camera_gaze_movement_identification.png create mode 100644 docs/img/aruco_camera_pose_estimation.png create mode 100644 docs/user_guide/aruco_markers_pipeline/aoi_analysis.md create mode 100644 docs/user_guide/aruco_markers_pipeline/aoi_projection.md delete mode 100644 docs/user_guide/aruco_markers_pipeline/aruco_camera_configuration_and_execution.md delete mode 100644 docs/user_guide/aruco_markers_pipeline/aruco_scene.md create mode 100644 docs/user_guide/aruco_markers_pipeline/configuration_and_execution.md create mode 100644 docs/user_guide/aruco_markers_pipeline/pose_estimation.md (limited to 'docs') diff --git a/docs/img/aruco_camera_aoi_projection.png b/docs/img/aruco_camera_aoi_projection.png new file mode 100644 index 0000000..59a8ab0 Binary files /dev/null and b/docs/img/aruco_camera_aoi_projection.png differ diff --git a/docs/img/aruco_camera_frame.png b/docs/img/aruco_camera_frame.png index 4b2a9a2..443285f 100644 Binary files a/docs/img/aruco_camera_frame.png and b/docs/img/aruco_camera_frame.png differ diff --git a/docs/img/aruco_camera_gaze_movement_identification.png b/docs/img/aruco_camera_gaze_movement_identification.png new file mode 100644 index 0000000..fc9ff39 Binary files /dev/null and b/docs/img/aruco_camera_gaze_movement_identification.png differ diff --git a/docs/img/aruco_camera_markers_detection.png b/docs/img/aruco_camera_markers_detection.png index 0452728..6192e09 100644 Binary files a/docs/img/aruco_camera_markers_detection.png and b/docs/img/aruco_camera_markers_detection.png differ diff --git a/docs/img/aruco_camera_pose_estimation.png b/docs/img/aruco_camera_pose_estimation.png new file mode 100644 index 0000000..b6c2675 Binary files /dev/null and b/docs/img/aruco_camera_pose_estimation.png differ diff --git a/docs/user_guide/aruco_markers_pipeline/aoi_analysis.md b/docs/user_guide/aruco_markers_pipeline/aoi_analysis.md new file mode 100644 index 0000000..05afcd3 --- /dev/null +++ b/docs/user_guide/aruco_markers_pipeline/aoi_analysis.md @@ -0,0 +1,58 @@ +Add AOI analysis +================ + + + +## Add AOIMatcher, AOIScanPath and AOIScanPathAnalyzers to ArUcoCamera layer to enable gaze analysis + +Here is the previous extract where AOI matcher, AOI scan path and AOI scan path analyzers are added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + ... + "scenes": { + "MyScene" : { + "aruco_markers_group": { + ... + }, + "layers": { + "MyLayer": { + "aoi_scene": { + ... + } + } + } + } + }, + "layers": { + "MyLayer": { + "aoi_matcher": { + "DeviationCircleCoverage": { + "coverage_threshold": 0.5 + } + }, + "aoi_scan_path": { + "duration_max": 30000 + }, + "aoi_scan_path_analyzers": { + "Basic": {}, + "TransitionMatrix": {}, + "NGram": { + "n_min": 3, + "n_max": 5 + } + } + } + }, + "gaze_movement_identifier": { + ... + } + ... +} +``` + +!!! warning + + Adding scan path and scan path analyzers to [an ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer doesn't make sense if the camera is moving. \ No newline at end of file diff --git a/docs/user_guide/aruco_markers_pipeline/aoi_description.md b/docs/user_guide/aruco_markers_pipeline/aoi_description.md index 80ad858..101ec9f 100644 --- a/docs/user_guide/aruco_markers_pipeline/aoi_description.md +++ b/docs/user_guide/aruco_markers_pipeline/aoi_description.md @@ -1,5 +1,5 @@ -Describe AOI scene -================== +Describe AOI +============ Once [ArUco markers are placed into a scene](aruco_markers_description.md), areas of interest need to be described into the same 3D referential. diff --git a/docs/user_guide/aruco_markers_pipeline/aoi_projection.md b/docs/user_guide/aruco_markers_pipeline/aoi_projection.md new file mode 100644 index 0000000..027f805 --- /dev/null +++ b/docs/user_guide/aruco_markers_pipeline/aoi_projection.md @@ -0,0 +1,159 @@ +Project AOI into camera frame +============================= + +Once [ArUcoScene pose is estimated](pose_estimation.md) and [AOI are described](aoi_description.md), AOI can be projected into [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame. + +![AOI projection](../../img/aruco_camera_aoi_projection.png) + +## Add ArLayer to ArUcoScene to load AOI + +The [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer) class allows to load areas of interest description. An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) instance can contains multiples [ArLayers](../../argaze.md/#argaze.ArFeatures.ArLayer). + +Here is the previous extract where one layer is added to the [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) configuration: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + ... + "scenes": { + "MyScene" : { + "aruco_markers_group": { + ... + }, + "layers": { + "MyLayer": { + "aoi_scene": { + "YellowSquare": [[6.2, -7.275252, 25.246159], [31.2, -7.275252, 25.246159], [31.2, 1.275252, 1.753843], [6.2, 1.275252, 1.753843]], + "GrayRectangle": [[2.5, 2.5, -0.5], [37.5, 2.5, -0.5], [37.5, 27.5, -0.5], [2.5, 27.5, -0.5]], + "BlueTriangle": [[12.5, 7.5, -0.5], [27.5, 7.5, -0.5], [20, 22.5, -0.5]] + } + } + } + } + } + ... +} +``` + +Now, let's understand the meaning of each JSON entry. + +### "MyLayer" + +The name of the [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer). Basically useful for visualisation purpose. + +### AOI Scene + +The [AOIScene](../../argaze.md/#argaze.AreaOfInterest.AOIFeatures.AOIScene) defines a set of 3D [AreaOfInterest](../../argaze.md/#argaze.AreaOfInterest.AOIFeatures.AreaOfInterest) registered by name. + +## Add ArLayer to ArUcoCamera to project AOI + +Here is the previous extract where one layer is added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) and displayed: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + ... + "scenes": { + "MyScene" : { + "aruco_markers_group": { + ... + }, + "layers": { + "MyLayer": { + "aoi_scene": { + ... + } + } + } + } + }, + "layers": { + "MyLayer": {} + } + ... + "image_parameters": { + ... + "draw_layers": { + "MyLayer": { + "draw_aoi_scene": { + "draw_aoi": { + "color": [255, 255, 255], + "border_size": 1 + } + } + } + } + } +} +``` + +Now, let's understand the meaning of each JSON entry. + +### "MyLayer" + +The name of the [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer). Basically useful for visualisation purpose. + +!!! warning "Layer name policy" + + An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) layer is projected into [an ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer, **provided they have the same name**. + +!!! note + + [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) layers are projected into their dedicated [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layers when calling the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method. + +## Add AOI analysis + +When a scene layer is projected into a camera layer, it means that the 3D [ArLayer.aoi_scene](../../argaze.md/#argaze.ArFeatures.ArLayer.aoi_scene) description of the scene becomes the 2D camera's [ArLayer.aoi_scene](../../argaze.md/#argaze.ArFeatures.ArLayer.aoi_scene) description of the camera. + +Therefore, it means that [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) benefits from all the services described in [AOI analysis pipeline section](../gaze_analysis_pipeline/aoi_analysis.md). + +Here is the previous extract where AOI matcher, AOI scan path and AOI scan path analyzers are added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + ... + "scenes": { + "MyScene" : { + "aruco_markers_group": { + ... + }, + "layers": { + "MyLayer": { + "aoi_scene": { + ... + } + } + } + } + }, + "layers": { + "MyLayer": { + "aoi_matcher": { + "DeviationCircleCoverage": { + "coverage_threshold": 0.5 + } + }, + "aoi_scan_path": { + "duration_max": 30000 + }, + "aoi_scan_path_analyzers": { + "Basic": {}, + "TransitionMatrix": {}, + "NGram": { + "n_min": 3, + "n_max": 5 + } + } + } + } + ... +} +``` + +!!! warning + + Adding scan path and scan path analyzers to an [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer doesn't make sense if the camera is moving. diff --git a/docs/user_guide/aruco_markers_pipeline/aruco_camera_configuration_and_execution.md b/docs/user_guide/aruco_markers_pipeline/aruco_camera_configuration_and_execution.md deleted file mode 100644 index ba19e45..0000000 --- a/docs/user_guide/aruco_markers_pipeline/aruco_camera_configuration_and_execution.md +++ /dev/null @@ -1,115 +0,0 @@ -Configure and execute ArUcoCamera -================================= - -Once [ArUco markers are placed into a scene](aruco_markers_description.md) and [areas of interest are described](aoi_description.md), everything is ready to setup an ArUco marker pipeline thanks to [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) class. - -As [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) inherits from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame), the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) class benefits from all the services described in [gaze analysis pipeline section](./user_guide/gaze_analysis_pipeline/introduction.md). - -![ArUco camera frame](../../img/aruco_camera_frame.png) - -## Load JSON configuration file - -The [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) internal pipeline loads from a JSON configuration file thanks to [ArUcoCamera.from_json](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera.from_json) class method. - -Here is a simple JSON [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) configuration file example: - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - "aruco_detector": { - "dictionary": "DICT_APRILTAG_16h5", - "marker_size": 5 - }, - "image_parameters": { - "background_weight": 1, - ... - "draw_detected_markers": { - "color": [0, 255, 0], - "draw_axes": { - "thickness": 3 - } - } - } -} -``` - -Then, here is how to load the JSON file: - -```python -from argaze.ArUcoMarkers import ArUcoCamera - -# Load ArUcoCamera -aruco_camera = ArUcoCamera.ArUcoCamera.from_json('./configuration.json') -``` - -Now, let's understand the meaning of each JSON entry. - -### Name - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* - -The name of the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame. Basically useful for visualisation purpose. - -### Size - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* - -The size of the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame in pixels. Be aware that gaze positions have to be in the same range of value to be projected in. - -### ArUco Detector - -The first [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) pipeline step is to detect ArUco markers inside input image and estimate their poses. - -The [ArUcoDetector](../../argaze.md/#argaze.ArUcoMarkers.ArUcoDetector) is in charge to detect all markers from a specific dictionary with a given size in centimeters. - -!!! warning - JSON *aruco_detector* entry is mandatory. - -### Image parameters - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* - -The usual [ArFrame visualisation parameters](./user_guide/gaze_analysis_pipeline/visualisation.md) plus one additional *draw_detected_markers* field. - -## Pipeline execution - -### Detect ArUco markers, estimate scene pose and project scene - -Pass each camera image to [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method to execute the whole pipeline dedicated to ArUco markers detection, scene pose estimation and projection. - -```python -# Assuming that Full HD (1920x1080) video stream or file is opened -... - -# Assuming that the video reading is handled in a looping code block -...: - - # Capture image from video stream of file - image = video_capture.read() - - # Detect ArUco markers, estimate scene pose then, project scene into camera frame - aruco_camera.watch(image) - - # Display ArUcoCamera frame image to check that ArUco markers are well detected and scene is well projected - ... aruco_camera.image() -``` - -!!! warning - ArUco markers pose estimation algorithm can lead to errors due to geometric ambiguities as explain in [this article](https://ieeexplore.ieee.org/document/1717461). To discard such ambiguous cases, markers should **not be parallel to camera plan**. - - - -### Analyse timestamped gaze positions into camera frame - -As mentioned above, [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) inherits from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame) and so, benefits from all the services described in [gaze analysis pipeline section](./user_guide/gaze_analysis_pipeline/introduction.md). - -Particularly, timestamped gaze positions can be passed one by one to [ArUcoCamera.look](../../argaze.md/#argaze.ArFeatures.ArFrame.look) method to execute the whole pipeline dedicated to gaze analysis. - -```python -# Assuming that timestamped gaze positions are available -... - - # Look ArUcoCamera frame at a timestamped gaze position - aruco_camera.look(timestamp, gaze_position) -``` - -!!! warning "" - - At this point, the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method only detects ArUco markers and the [ArUcoCamera.look](../../argaze.md/#argaze.ArFeatures.ArCamera.look) method is not able to analyze gaze positions as no scene description is provided into the JSON configuration file. - - Read the next chapters to learn [how to enable scene pose estimation and its projection](aruco_scene.md). \ No newline at end of file diff --git a/docs/user_guide/aruco_markers_pipeline/aruco_scene.md b/docs/user_guide/aruco_markers_pipeline/aruco_scene.md deleted file mode 100644 index 91d2702..0000000 --- a/docs/user_guide/aruco_markers_pipeline/aruco_scene.md +++ /dev/null @@ -1,245 +0,0 @@ -Add an ArUcoScene -================= - -An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) class defines a space where [areas of interest description](aoi_description.md) is mapped onto [ArUco markers description](aruco_markers_description.md). - -![ArUco scene](../../img/aruco_scene.png) - -## Add ArUcoScene to ArUcoCamera JSON configuration file - -An [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) instance can contains multiples [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene). - -Here is an extract from the JSON [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) configuration file with a sample where one scene is added: - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - ... - "scenes": { - "MyScene" : { - "aruco_markers_group": { - "dictionary": "DICT_APRILTAG_16h5", - "marker_size": 5, - "places": { - "0": { - "translation": [2.5, 2.5, 0], - "rotation": [0, 0, 0] - }, - "1": { - "translation": [0, 25.5, 2], - "rotation": [0, 45, 0] - }, - "2": { - "translation": [35.5, 0, 3], - "rotation": [-45, 0, 0] - } - } - } - } - } - ... -} -``` - -Now, let's understand the meaning of each JSON entry. - -### "MyScene" - -The name of the [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene). Basically useful for visualisation purpose. - -### ArUco markers group - -The 3D places of ArUco markers into the scene as defined at [ArUco markers description chapter](aruco_markers_description.md). Thanks to this description, it is possible to estimate the pose of [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) in [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame. - -!!! note - - [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) pose estimation is done when calling the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method. - -## Add ArLayer to ArUcoScene to load AOI - -The [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer) class allows to load areas of interest description. An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) instance can contains multiples [ArLayers](../../argaze.md/#argaze.ArFeatures.ArLayer). - -Here is the previous extract where one layer is added to the [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene): - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - ... - "scenes": { - "MyScene" : { - "aruco_markers_group": { - ... - }, - "layers": { - "MyLayer": { - "aoi_scene": { - "YellowSquare": [[6.2, -7.275252, 25.246159], [31.2, -7.275252, 25.246159], [31.2, 1.275252, 1.753843], [6.2, 1.275252, 1.753843]], - "GrayRectangle": [[2.5, 2.5, -0.5], [37.5, 2.5, -0.5], [37.5, 27.5, -0.5], [2.5, 27.5, -0.5]], - "BlueTriangle": [[12.5, 7.5, -0.5], [27.5, 7.5, -0.5], [20, 22.5, -0.5]] - } - } - } - } - } - ... -} -``` - -Now, let's understand the meaning of each JSON entry. - -### "MyLayer" - -The name of the [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer). Basically useful for visualisation purpose. - -### AOI Scene - -The [AOIScene](../../argaze.md/#argaze.AreaOfInterest.AOIFeatures.AOIScene) defines a set of 3D [AreaOfInterest](../../argaze.md/#argaze.AreaOfInterest.AOIFeatures.AreaOfInterest) registered by name. - -## Add ArLayer to ArUcoCamera to project AOI - -Here is the previous extract where one layer is added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera): - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - ... - "scenes": { - "MyScene" : { - "aruco_markers_group": { - ... - }, - "layers": { - "MyLayer": { - "aoi_scene": { - ... - } - } - } - } - }, - "layers": { - "MyLayer": { - "aoi_matcher": { - ... - }, - "aoi_scan_path": { - "duration_max": 30000 - }, - "aoi_scan_path_analyzers": { - ... - } - } - } - ... -} -``` - -Now, let's understand the meaning of each JSON entry. - -### "MyLayer" - -The name of the [ArLayer](../../argaze.md/#argaze.ArFeatures.ArLayer). Basically useful for visualisation purpose. - -!!! warning - - An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) layer is projected into [an ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer, **provided they have the same name**. - -!!! note - - [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) layers are projected into their dedicated [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layers when calling the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method. - -## Add GazeMovementIdentifier to ArUcocamera to enable gaze movement identification - -Here is the previous extract where gaze movement identifier is added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera): - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - ... - "scenes": { - "MyScene" : { - "aruco_markers_group": { - ... - }, - "layers": { - "MyLayer": { - "aoi_scene": { - ... - } - } - } - } - }, - "layers": { - "MyLayer": {} - }, - "gaze_movement_identifier": { - "DispersionThresholdIdentification": { - "deviation_max_threshold": 50, - "duration_min_threshold": 200 - } - } - ... -} -``` -!!! note - - Timestamped gaze positions are [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layers execution is done when calling the [ArUcoCamera.look](../../argaze.md/#argaze.ArFeatures.ArCamera.look) method. - -## Add AOIMatcher, AOIScanPath and AOIScanPathAnalyzers to ArUcoCamera layer to enable gaze analysis - -Here is the previous extract where AOI matcher, AOI scan path and AOI scan path analyzers are added to the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer: - -```json -{ - "name": "My FullHD camera", - "size": [1920, 1080], - ... - "scenes": { - "MyScene" : { - "aruco_markers_group": { - ... - }, - "layers": { - "MyLayer": { - "aoi_scene": { - ... - } - } - } - } - }, - "layers": { - "MyLayer": { - "aoi_matcher": { - "DeviationCircleCoverage": { - "coverage_threshold": 0.5 - } - }, - "aoi_scan_path": { - "duration_max": 30000 - }, - "aoi_scan_path_analyzers": { - "Basic": {}, - "TransitionMatrix": {}, - "NGram": { - "n_min": 3, - "n_max": 5 - } - } - } - }, - "gaze_movement_identifier": { - ... - } - ... -} -``` - -!!! warning - - Adding scan path and scan path analyzers to [an ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) layer doesn't make sense if the camera is moving. \ No newline at end of file diff --git a/docs/user_guide/aruco_markers_pipeline/configuration_and_execution.md b/docs/user_guide/aruco_markers_pipeline/configuration_and_execution.md new file mode 100644 index 0000000..81c577f --- /dev/null +++ b/docs/user_guide/aruco_markers_pipeline/configuration_and_execution.md @@ -0,0 +1,138 @@ +Load and execute pipeline +========================= + +Once [ArUco markers are placed into a scene](aruco_markers_description.md), they can be detected thanks to [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) class. + +As [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) inherits from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame), the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) class also benefits from all the services described in [gaze analysis pipeline section](./user_guide/gaze_analysis_pipeline/introduction.md). + +![ArUco camera frame](../../img/aruco_camera_frame.png) + +## Load JSON configuration file + +The [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) internal pipeline loads from a JSON configuration file thanks to [ArUcoCamera.from_json](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera.from_json) class method. + +Here is a simple JSON [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) configuration file example: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + "aruco_detector": { + "dictionary": "DICT_APRILTAG_16h5", + "marker_size": 5 + }, + "gaze_movement_identifier": { + "DispersionThresholdIdentification": { + "deviation_max_threshold": 25, + "duration_min_threshold": 150 + } + }, + "image_parameters": { + "background_weight": 1, + "draw_gaze_positions": { + "color": [0, 255, 255], + "size": 2 + }, + "draw_fixations": { + "deviation_circle_color": [255, 0, 255], + "duration_border_color": [127, 0, 127], + "duration_factor": 1e-2 + }, + "draw_saccades": { + "line_color": [255, 0, 255] + }, + "draw_detected_markers": { + "color": [0, 255, 0], + "draw_axes": { + "thickness": 3 + } + } + } +} +``` + +Then, here is how to load the JSON file: + +```python +from argaze.ArUcoMarkers import ArUcoCamera + +# Load ArUcoCamera +aruco_camera = ArUcoCamera.ArUcoCamera.from_json('./configuration.json') +``` + +Now, let's understand the meaning of each JSON entry. + +### Name - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* + +The name of the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame. Basically useful for visualisation purpose. + +### Size - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* + +The size of the [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame in pixels. Be aware that gaze positions have to be in the same range of value to be projected in. + +### ArUco Detector + +The first [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) pipeline step is to detect ArUco markers inside input image and estimate their poses. + +![ArUco markers detection](../../img/aruco_camera_markers_detection.png) + +The [ArUcoDetector](../../argaze.md/#argaze.ArUcoMarkers.ArUcoDetector) is in charge to detect all markers from a specific dictionary with a given size in centimeters. + +!!! warning "Mandatory" + JSON *aruco_detector* entry is mandatory. + +### Gaze Movement Identifier - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* + +The first [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame) pipeline step dedicated to identify fixations or saccades from consecutive timestamped gaze positions. + +![Gaze movement identification](../../img/aruco_camera_gaze_movement_identification.png) + +### Image parameters - *inherited from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame)* + +The usual [ArFrame visualisation parameters](./user_guide/gaze_analysis_pipeline/visualisation.md) plus one additional *draw_detected_markers* field. + +## Pipeline execution + +### Detect ArUco markers, estimate scene pose and project AOI + +Pass each camera image to [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method to execute the whole pipeline dedicated to ArUco markers detection, scene pose estimation and AOI projection. + +```python +# Assuming that Full HD (1920x1080) video stream or file is opened +... + +# Assuming that the video reading is handled in a looping code block +...: + + # Capture image from video stream of file + image = video_capture.read() + + # Detect ArUco markers, estimate scene pose then, project AOI into camera frame + aruco_camera.watch(image) + + # Display ArUcoCamera frame image to display detected ArUco markers, scene pose, AOI projection and ArFrame visualisation. + ... aruco_camera.image() +``` + +!!! warning "Pose estimation error" + ArUco markers pose estimation algorithm can lead to errors due to geometric ambiguities as explain in [this article](https://ieeexplore.ieee.org/document/1717461). To discard such ambiguous cases, markers should **as less as possible be parallel to camera plan**. + +### Analyse timestamped gaze positions into camera frame + +As mentioned above, [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) inherits from [ArFrame](../../argaze.md/#argaze.ArFeatures.ArFrame) and so, benefits from all the services described in [gaze analysis pipeline section](./user_guide/gaze_analysis_pipeline/introduction.md). + +Particularly, timestamped gaze positions can be passed one by one to [ArUcoCamera.look](../../argaze.md/#argaze.ArFeatures.ArFrame.look) method to execute the whole pipeline dedicated to gaze analysis. + +```python +# Assuming that timestamped gaze positions are available +... + + # Look ArUcoCamera frame at a timestamped gaze position + aruco_camera.look(timestamp, gaze_position) +``` + +!!! note "" + + At this point, the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method only detects ArUco markers and the [ArUcoCamera.look](../../argaze.md/#argaze.ArFeatures.ArCamera.look) method only process gaze movement identification without any AOI support as no scene description is provided into the JSON configuration file. + + Read the next chapters to learn [how to estimate scene pose](pose_estimation.md) and [how to project AOI](aoi_projection.md). \ No newline at end of file diff --git a/docs/user_guide/aruco_markers_pipeline/introduction.md b/docs/user_guide/aruco_markers_pipeline/introduction.md index a10ca28..836569a 100644 --- a/docs/user_guide/aruco_markers_pipeline/introduction.md +++ b/docs/user_guide/aruco_markers_pipeline/introduction.md @@ -16,9 +16,10 @@ First, let's look at the schema below: it gives an overview of the main notions To build your own ArUco markers pipeline, you need to know: * [How to setup ArUco markers into a scene](aruco_markers_description.md), -* [How to describe areas of interest in the same scene](aoi_description.md), -* [How to deal with an ArUcoCamera instance](aruco_camera_configuration_and_execution.md), -* [How to add ArUcoScene instance](aruco_scene.md), +* [How to describe scene's AOI](aoi_description.md), +* [How to load and execute ArUco markers pipeline](configuration_and_execution.md), +* [How to estimate scene pose](pose_estimation.md), +* [How to project AOI into camera frame](aoi_projection.md), * [How to visualize ArUcoCamera and ArUcoScenes](visualisation.md) More advanced features are also explained like: diff --git a/docs/user_guide/aruco_markers_pipeline/pose_estimation.md b/docs/user_guide/aruco_markers_pipeline/pose_estimation.md new file mode 100644 index 0000000..6acafee --- /dev/null +++ b/docs/user_guide/aruco_markers_pipeline/pose_estimation.md @@ -0,0 +1,82 @@ +Estimate scene pose +=================== + +An [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) class defines a space with [ArUco markers inside](aruco_markers_description.md) helping to estimate scene pose when they are watched by [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera). + +![Scene pose estimation](../../img/aruco_camera_pose_estimation.png) + +## Add ArUcoScene to ArUcoCamera JSON configuration file + +An [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) instance can contains multiples [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene). + +Here is an extract from the JSON [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) configuration file with a sample where one scene is added and displayed: + +```json +{ + "name": "My FullHD camera", + "size": [1920, 1080], + ... + "scenes": { + "MyScene" : { + "aruco_markers_group": { + "dictionary": "DICT_APRILTAG_16h5", + "marker_size": 5, + "places": { + "0": { + "translation": [2.5, 2.5, 0], + "rotation": [0, 0, 0] + }, + "1": { + "translation": [0, 25.5, 2], + "rotation": [0, 45, 0] + }, + "2": { + "translation": [35.5, 0, 3], + "rotation": [-45, 0, 0] + } + } + } + } + } + ... + "image_parameters": { + ... + "draw_scenes": { + "MyScene": { + "draw_aruco_markers_group": { + "draw_axes": { + "thickness": 3, + "length": 10 + }, + "draw_places": { + "color": [0, 0, 0], + "border_size": 1 + }, + "draw_places_axes": { + "thickness": 1, + "length": 2.5 + } + } + } + } + } +} +``` + +Now, let's understand the meaning of each JSON entry. + +### "MyScene" + +The name of the [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene). Basically useful for visualisation purpose. + +### ArUco markers group + +The 3D places of ArUco markers into the scene as defined at [ArUco markers description chapter](aruco_markers_description.md). Thanks to this description, it is possible to estimate the pose of [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) in [ArUcoCamera](../../argaze.md/#argaze.ArUcoMarkers.ArUcoCamera) frame. + +!!! note + + [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) pose estimation is done when calling the [ArUcoCamera.watch](../../argaze.md/#argaze.ArFeatures.ArCamera.watch) method. + +### Draw scenes + +The drawing parameters of each loaded [ArUcoScene](../../argaze.md/#argaze.ArUcoMarkers.ArUcoScene) in [ArUcoCamera.image](../../argaze.md/#argaze.ArFeatures.ArFrame.image). -- cgit v1.1