diff --git a/data_samples/json_conf_files/a_basic_pipeline.json b/data_samples/json_conf_files/a_basic_pipeline.json index ea154f3..ef5b582 100644 --- a/data_samples/json_conf_files/a_basic_pipeline.json +++ b/data_samples/json_conf_files/a_basic_pipeline.json @@ -21,7 +21,7 @@ "invalid_disparity": "NaN" }, "refinement": { - "refinement_method": "interpolation" + "refinement_method": "optical_flow" } } } diff --git a/data_samples/json_conf_files/an_estimation_pipeline.json b/data_samples/json_conf_files/an_estimation_pipeline.json new file mode 100644 index 0000000..1b027fd --- /dev/null +++ b/data_samples/json_conf_files/an_estimation_pipeline.json @@ -0,0 +1,31 @@ +{ + "input": { + "left": { + "img": "./maricopa/left.tif", + "nodata": -9999 + }, + "right": { + "img": "./maricopa/right.tif", + "nodata": -9999 + } + }, + "pipeline": { + "estimation": { + "estimation_method": "phase_cross_correlation", + "range_row": 5, + "range_col": 5, + "sample_factor": 100 + }, + "matching_cost": { + "matching_cost_method": "zncc", + "window_size": 5 + }, + "disparity": { + "disparity_method": "wta", + "invalid_disparity": "NaN" + }, + "refinement": { + "refinement_method": "interpolation" + } + } +} diff --git a/docs/source/Images/Pandora2D_pipeline.png b/docs/source/Images/Pandora2D_pipeline.png index ec1b012..81866f8 100644 Binary files a/docs/source/Images/Pandora2D_pipeline.png and b/docs/source/Images/Pandora2D_pipeline.png differ diff --git a/docs/source/Images/estimation_schema.png b/docs/source/Images/estimation_schema.png new file mode 100644 index 0000000..2c50417 Binary files /dev/null and b/docs/source/Images/estimation_schema.png differ diff --git a/docs/source/Images/img_pipeline/dp_step.png b/docs/source/Images/img_pipeline/dp_step.png index 203a084..a13de3e 100644 Binary files a/docs/source/Images/img_pipeline/dp_step.png and b/docs/source/Images/img_pipeline/dp_step.png differ diff --git a/docs/source/Images/img_pipeline/estimation_step.png b/docs/source/Images/img_pipeline/estimation_step.png new file mode 100644 index 0000000..32941f2 Binary files /dev/null and b/docs/source/Images/img_pipeline/estimation_step.png differ diff --git a/docs/source/Images/img_pipeline/mc_step.png b/docs/source/Images/img_pipeline/mc_step.png index 846d1cd..13dfbec 100644 Binary files a/docs/source/Images/img_pipeline/mc_step.png and b/docs/source/Images/img_pipeline/mc_step.png differ diff --git a/docs/source/Images/img_pipeline/refi_step.png b/docs/source/Images/img_pipeline/refi_step.png index c4c677f..595ede0 100644 Binary files a/docs/source/Images/img_pipeline/refi_step.png and b/docs/source/Images/img_pipeline/refi_step.png differ diff --git a/docs/source/Images/optical_flow_schema.png b/docs/source/Images/optical_flow_schema.png new file mode 100644 index 0000000..937315c Binary files /dev/null and b/docs/source/Images/optical_flow_schema.png differ diff --git a/docs/source/Images/range_schema.png b/docs/source/Images/range_schema.png new file mode 100644 index 0000000..6d9cdcc Binary files /dev/null and b/docs/source/Images/range_schema.png differ diff --git a/docs/source/exploring_the_field.rst b/docs/source/exploring_the_field.rst new file mode 100644 index 0000000..c632115 --- /dev/null +++ b/docs/source/exploring_the_field.rst @@ -0,0 +1,9 @@ +Exploring the field +=================== + +.. toctree:: + :maxdepth: 2 + + exploring_the_field/initial_disparity.rst + exploring_the_field/refining_disparity.rst + diff --git a/docs/source/exploring_the_field/initial_disparity.rst b/docs/source/exploring_the_field/initial_disparity.rst new file mode 100644 index 0000000..1e011d0 --- /dev/null +++ b/docs/source/exploring_the_field/initial_disparity.rst @@ -0,0 +1,38 @@ +.. _initial_disparity: + +Disparity range exploration +=========================== + +The user is required to set up pandora2d by specifying a range of disparity to be explored. +There are two available methods to do this. + +Setting an interval +------------------- + +In the configuration file, the user is required to enter disparity range, as a list with two elements each, indicating +the minimum and maximum values for both row and columns disparity. + +.. code:: json + :name: Setting disparity ranges example + + { + "input": + { + "col_disparity": [-2, 2], + "row_disparity": [-2, 2] + } + } + + +.. figure:: ../Images/range_schema.png + + +Setting a range +--------------- + +In situations where the user does not know the required interval range, an alternative method is provided. +The user should leave the 'disparity_col' and 'disparity_row' parameters empty. Instead, they need to enable an estimation stage in the pipeline. This stage calculates a global shift throughout the image. By using 'range_col' and 'range_row' parameters, the user can then approximate an interval around this determined shift. + +The following diagram illustrates how the disparity intervals are initialized using the estimation step: + +.. figure:: ../Images/estimation_schema.png diff --git a/docs/source/exploring_the_field/refining_disparity.rst b/docs/source/exploring_the_field/refining_disparity.rst new file mode 100644 index 0000000..5408c02 --- /dev/null +++ b/docs/source/exploring_the_field/refining_disparity.rst @@ -0,0 +1,15 @@ +.. _refining disparity: + +Refinement step +=============== +The purpose of this step is to refine the disparity identified in the previous step. +So, the refinement step involves transforming a pixel disparity map into a sub-pixel disparity map. + + +Two methods are available in pandora2d: + +- Interpolation +- Optical flow. + +.. warning:: + The optical flow method is still in an experimental phase. diff --git a/docs/source/index.rst b/docs/source/index.rst index 5b41fd3..638c593 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,10 +15,12 @@ Welcome to Pandora2D's documentation! :caption: Contents: getting_started + exploring_the_field userguide api_reference/index.rst developer_guide + .. api_reference is automatically generated by sphinx-autoapi diff --git a/docs/source/userguide/as_an_api.rst b/docs/source/userguide/as_an_api.rst index ee4cf0f..b198a7d 100644 --- a/docs/source/userguide/as_an_api.rst +++ b/docs/source/userguide/as_an_api.rst @@ -39,12 +39,6 @@ Pandora2D provides a full python API which can be used to compute disparity maps } } - # read images - image_datasets = create_datasets_from_inputs(input_config=image_cfg["input"]) - - # instantiate Pandora2D Machine - pandora2d_machine = Pandora2DMachine() - # define pipeline configuration user_pipeline_cfg = { 'pipeline':{ @@ -57,11 +51,17 @@ Pandora2D provides a full python API which can be used to compute disparity maps "invalid_disparity": -9999 }, "refinement" : { - "refinement_method" : "interpolation" + "refinement_method" : "optical_flow" } } } + # read images + image_datasets = create_datasets_from_inputs(input_config=image_cfg["input"], estimation_cfg=user_pipeline_cfg.get("estimation")) + + # instantiate Pandora2D Machine + pandora2d_machine = Pandora2DMachine() + # check the configurations and sequences steps pipeline_cfg = check_configuration.check_pipeline_section(user_pipeline_cfg, pandora2d_machine) @@ -90,7 +90,7 @@ Images ###### Pandora2D reads the input images before the stereo computation and creates two datasets, one for the left and one for the right -image which contain the data's image, data's mask and additionnal information. +image which contain the data's image and additional information. Example of an image dataset diff --git a/docs/source/userguide/input.rst b/docs/source/userguide/input.rst index e23c4a0..96d1502 100644 --- a/docs/source/userguide/input.rst +++ b/docs/source/userguide/input.rst @@ -33,12 +33,12 @@ Input section is composed of the following keys: - Minimal and Maximal disparities for columns - [int, int] - - - Yes + - If the estimation step is not present * - *row_disparity* - Minimal and Maximal disparities for rows - [int, int] - - - Yes + - If the estimation step is not present Left and Right properties are composed of the following keys: diff --git a/docs/source/userguide/output.rst b/docs/source/userguide/output.rst index a87a9ad..fc8f8ae 100644 --- a/docs/source/userguide/output.rst +++ b/docs/source/userguide/output.rst @@ -14,4 +14,4 @@ Saved images Saved configuration ******************* -- ./res/cfg/config.json : the config file used to run Pandora2D \ No newline at end of file +- ./res/cfg/config.json : the config file used to run Pandora2D and estimation information if computed \ No newline at end of file diff --git a/docs/source/userguide/overview.rst b/docs/source/userguide/overview.rst index 9a9275a..a1ccb82 100644 --- a/docs/source/userguide/overview.rst +++ b/docs/source/userguide/overview.rst @@ -14,6 +14,13 @@ The following interactive diagram highlights all steps available in Pandora2D. .. image:: ../Images/img_pipeline/arrow.png :align: center +.. image:: ../Images/img_pipeline/estimation_step.png + :align: center + :target: step_by_step/refinement.html + +.. image:: ../Images/img_pipeline/arrow.png + :align: center + .. image:: ../Images/img_pipeline/mc_step.png :align: center :target: step_by_step/matching_cost.html @@ -44,11 +51,6 @@ The following interactive diagram highlights all steps available in Pandora2D. forced line break, -.. note:: - - Dark red blocks represent mandatory steps. - - Pink blocks represent optional steps. - - Configuration file ****************** @@ -114,7 +116,7 @@ Example "invalid_disparity": -999 }, "refinement": { - "refinement_method": "interpolation" + "refinement_method": "optical_flow" } } } diff --git a/docs/source/userguide/sequencing.rst b/docs/source/userguide/sequencing.rst index a3d4f64..79141fc 100644 --- a/docs/source/userguide/sequencing.rst +++ b/docs/source/userguide/sequencing.rst @@ -7,10 +7,12 @@ transition defined by the Pandora2D Machine (`transitions `__ + + +Configuration and parameters +---------------------------- +.. warning:: + + You don't need to set disparities in input section if you set the estimation step + +.. list-table:: Parameters + :header-rows: 1 + + + * - Name + - Description + - Type + - Default value + - Available value + - Required + * - *estimation_method* + - estimation measure + - string + - + - "phase_cross_correlation" + - Yes + * - *range_col* + - Exploration around the initial disparity for columns + - int + - 5 + - >0, odd number + - No + * - *range_row* + - Exploration around the initial disparity for rows + - int + - 5 + - >0, odd number + - No + * - *sample_factor* + - | Upsampling factor. + | Images will be registered to within 1 / upsample_factor of a pixel + - int + - 1 + - >= 1 + - No + + +**Example** + +.. sourcecode:: text + + { + "input" : + { + ... + }, + "pipeline" : + { + "estimation": + { + "estimation_method": "phase_cross_correlation", + "range_col": 5, + "range_row": 5, + "sample_factor": 20 + } + ... + } + } + + +Outputs: +-------- + +- Showed in log in verbose mode +- Written in the output configuration file +- Stored in the inputs_dataset \ No newline at end of file diff --git a/docs/source/userguide/step_by_step/refinement.rst b/docs/source/userguide/step_by_step/refinement.rst index e388c7c..6f321e6 100644 --- a/docs/source/userguide/step_by_step/refinement.rst +++ b/docs/source/userguide/step_by_step/refinement.rst @@ -2,25 +2,102 @@ Refinement of the disparity maps ================================ - -Theoretical basics ------------------- The purpose of this step is to refine the disparity identified in the previous step. -The available refinement methods are: - * **Interpolation**: +Interpolation method +-------------------- + +It consists on 3 different steps: + + * First, the cost_volumes is reshaped to obtain the 2D (disp_row, disp_col) costs map for each pixel, so we will obtain (row * col) 2D cost maps. + * The cost map of each pixel is interpolated using scipy to obtain a continuous function. + * Then, the interpolated functions are minimized using scipy to obtain the refined disparities. + +Optical_flow method +------------------- +.. warning:: + The optical flow method is still in an experimental phase. + +Inspired by [Lucas & Kanade]_.'s algorithm + + * We first need to suppose that pixel's shifting are subpixel between left and right images. + * Second, we need to suppose brightness constancy between left and right images. (2) + * Now, we can write : + + .. math:: + + I(x, y, t) &= I(x + dx, y + dy, t + dt) \\ + I(x, y, t) &= I(x, y, t) + \frac{\partial I}{\partial x}\partial x + \frac{\partial I}{\partial y}\partial y +\frac{\partial I}{\partial t}\partial t + + with hypothesis (2) : + + .. math:: + + \frac{\partial I}{\partial x} dx + \frac{\partial I}{\partial y} dy + \frac{\partial I}{\partial t}dt = 0 + + after dividing by :math:`dt`: + + .. math:: + + \frac{\partial I}{\partial x} \frac{dx}{dt} + \frac{\partial I}{\partial y} \frac{dy}{dt} = - \frac{\partial I}{\partial t} + + * We can resolve v thanks to least squares method : + + .. math:: + + v = (A^T A)^{-1}A^T B + + * Lucas & Kanade works on a pixel and his neighbourhood so : + + .. math:: + + A = + \left(\begin{array}{cc} + I_x(q1) & I_y(q1)\\ + I_x(q2) & I_y(q2) \\ + . & . \\ + . & . \\ + . & . \\ + I_x(qn) & I_y(qn) + \end{array}\right) + + v = + \left(\begin{array}{cc} + V_x\\ + V_y + \end{array}\right) + + + B = + \left(\begin{array}{cc} + -I_t(q1) \\ + -I_t(q2) \\ + . \\ + . \\ + . \\ + -I_t(qn) + \end{array}\right) + +The following diagram presents the different steps implemented in Pandora2d to enable +the refinement of the disparity map with optical flow. + +.. [Lucas & Kanade] An iterative image registration technique with an application to stereo vision. + Proceedings of Imaging Understanding Workshop, pages 121--130. + +.. figure:: ../../Images/optical_flow_schema.png + :width: 1000px + :height: 200px + +Dichotomy method +---------------- + +It’s an iterative process that will, at each iteration: + * compute the half way positions between each best candidate in the cost volume and its nearest neighbours. + * compute the similarity coefficients at those positions using the given filter method. + * find the new best candidate from those computed coefficients. - It consists on 3 different steps: - * First, the cost_volumes is reshaped to obtain the 2D (disp_row, disp_col) costs map for each pixel, so we will obtain (row * col) 2D cost maps. - * The cost map of each pixel is interpolated using scipy to obtain a continuous function. - * Then, the interpolated functions are minimized using scipy to obtain the refined disparities. - * **Dichotomy**: - It’s an iterative process that will, at each iteration: - * compute the half way positions between each best candidate in the cost volume and its nearest neighbours. - * compute the similarity coefficients at those positions using the given filter method. - * find the new best candidate from those computed coefficients. Configuration and parameters ---------------------------- @@ -34,21 +111,27 @@ Configuration and parameters - Default value - Available value - Required - * - *refinemement_method* + * - *refinement_method* - Refinement method - string - - | "interpolation", | "dichotomy", + | "optical_flow" - Yes * - *iterations* - - Number of iterations + - Number of iterations (not available for interpolation) - integer - - - - | 1 to 9 + - 4 for **optical_flow** method + - | **Dichotomy** + | 1 to 9 | *if above, will be bound to 9* - | **Only available if "dichotomy" method** - - Yes + | **Optical flow** + | >0 + - | **Dichotomy** + | Yes + | **Optical flow** + | No * - *filter* - Name of the filter to use - str @@ -74,7 +157,7 @@ Configuration and parameters // ... "refinement": { - "refinement_method": "interpolation" + "refinement_method": "optical_flow" }, // ... } diff --git a/notebooks/estimation_step_explained.ipynb b/notebooks/estimation_step_explained.ipynb new file mode 100644 index 0000000..1b05170 --- /dev/null +++ b/notebooks/estimation_step_explained.ipynb @@ -0,0 +1,342 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "db04320a-5fb9-47ca-8dcc-248e00941aa8", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "id": "f15f0456-611e-451f-87e6-dcf58e1ef190", + "metadata": {}, + "source": [ + "# Pandora2D : a coregistration framework" + ] + }, + { + "cell_type": "markdown", + "id": "15158305-7b79-4263-aeaa-16b39235f7e8", + "metadata": {}, + "source": [ + "# Introduction and basic usage" + ] + }, + { + "cell_type": "markdown", + "id": "e8458dd9-7977-4695-bdb5-047c47eec0b0", + "metadata": {}, + "source": [ + "#### Imports and external functions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac793218-8f2b-4559-9ef4-bcb6f917a59b", + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "from pathlib import Path\n", + "from IPython.display import Image, display\n", + "from pprint import pprint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbbf6721-9115-44fb-9525-8e6d81938355", + "metadata": {}, + "outputs": [], + "source": [ + "from snippets.utils import *" + ] + }, + { + "cell_type": "markdown", + "id": "e4e477f5-6a2d-40c6-b747-7cf00791decb", + "metadata": {}, + "source": [ + "## Pandora2D's pipeline\n", + "\n", + "Pandora2D provides the following steps:\n", + "* estimation computation\n", + "* matching cost computation\n", + "* disparity computation (**mandatory if matching_cost**)\n", + "* subpixel disparity refinement" + ] + }, + { + "cell_type": "markdown", + "id": "1be05d02-b1dc-43ec-a09a-d65c8285ea44", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "id": "fdaacb98-cbac-4647-867c-ef110f17a85c", + "metadata": {}, + "source": [ + "## About the estimation step :" + ] + }, + { + "cell_type": "markdown", + "id": "25de08c8-52ab-436f-9bc5-c9b665e672cb", + "metadata": {}, + "source": [ + "1. used alone, it computes a constant a priori offset across the entire image in line and column. \n", + "\n", + "2. When used in a complete pipeline, it enables, **thanks to an interval entered by the user**, the calculation of \"col_disparity\" and \"row_disparity\" parameters in the input configuration dictionnary." + ] + }, + { + "cell_type": "markdown", + "id": "fa43b579-4d68-4c78-8a07-6c6ee05805e5", + "metadata": {}, + "source": [ + "The outputs are : \n", + "- shown in log in verbose mode\n", + "- saved in the configuration file\n", + "- stored in dataset image" + ] + }, + { + "cell_type": "markdown", + "id": "00469662-a597-4976-88df-1cd6b9c30b77", + "metadata": {}, + "source": [ + "# Pandora2D execution options with state machine" + ] + }, + { + "cell_type": "markdown", + "id": "548a5fd1-4c3b-4dc7-bba3-83114f84396b", + "metadata": {}, + "source": [ + "#### Imports of pandora2d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4fabcfd6-e078-491c-86e9-407ff5eca137", + "metadata": {}, + "outputs": [], + "source": [ + "# Load pandora2d imports\n", + "import pandora2d\n", + "from pandora2d.state_machine import Pandora2DMachine\n", + "from pandora2d.check_configuration import check_conf\n", + "from pandora2d.img_tools import create_datasets_from_inputs" + ] + }, + { + "cell_type": "markdown", + "id": "21cc33a4-0264-443c-97c8-ad32b80d9c65", + "metadata": {}, + "source": [ + "#### Load and visualize input data " + ] + }, + { + "cell_type": "markdown", + "id": "6d44265d-decd-4157-817f-40bd1a3df0a5", + "metadata": {}, + "source": [ + "Provide image path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8df92f0f-298e-4f1d-b5a2-f7bbf0622646", + "metadata": {}, + "outputs": [], + "source": [ + "# Paths to left and right images\n", + "img_left_path = \"data/left.tif\"\n", + "img_right_path = \"data/right.tif\"" + ] + }, + { + "cell_type": "markdown", + "id": "bfb900c7-9356-48be-9a3a-74e1ddb5b27e", + "metadata": {}, + "source": [ + "#### No need for disparity Range in the estimation process." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22aed1c4-a53a-43bd-bbd3-10eb64b59e63", + "metadata": {}, + "outputs": [], + "source": [ + "user_cfg = {\n", + " \"input\": {\n", + " \"left\": {\n", + " \"img\": img_left_path, \n", + " \"nodata\": np.nan\n", + " },\n", + " \"right\": {\n", + " \"img\": img_right_path, \n", + " \"nodata\": np.nan\n", + " },\n", + " },\n", + " \"pipeline\": { \n", + " \"estimation\": {\n", + " \"estimation_method\": \"phase_cross_correlation\",\n", + " \"range_row\": 5, \n", + " \"range_col\": 5,\n", + " \"sample_factor\": 100\n", + " },\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2931b134-aaee-4086-a6b4-88cec3bc39f3", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "img_left, img_right = create_datasets_from_inputs(input_config=user_cfg[\"input\"], estimation_cfg=user_cfg[\"pipeline\"].get(\"estimation\"))" + ] + }, + { + "cell_type": "markdown", + "id": "4def9934-12f8-42cd-a05b-bc845a1a1620", + "metadata": {}, + "source": [ + "#### Instantiate the machine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47546952-4125-43b9-aaa1-71fe48cac2d8", + "metadata": {}, + "outputs": [], + "source": [ + "pandora2d_machine = Pandora2DMachine()" + ] + }, + { + "cell_type": "markdown", + "id": "d69483f3-7a28-4530-b286-ec40c64ecfb2", + "metadata": {}, + "source": [ + "#### Check the configuration and sequence of steps" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c085399b-eb82-4614-8135-3bf4bbcbf583", + "metadata": {}, + "outputs": [], + "source": [ + "checked_cfg = check_conf(user_cfg, pandora2d_machine)" + ] + }, + { + "cell_type": "markdown", + "id": "4cb3efad-3e1c-4c38-b0e1-2405d7431f3f", + "metadata": {}, + "source": [ + "#### Trigger all the steps of the machine at ones" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efc291e7-03ef-4fa4-9aed-7eee6f12639a", + "metadata": {}, + "outputs": [], + "source": [ + "dataset, estimation_results = pandora2d.run(\n", + " pandora2d_machine,\n", + " img_left,\n", + " img_right,\n", + " checked_cfg\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "c043bc01-12c8-4caf-b65d-1d46bcca90fb", + "metadata": {}, + "source": [ + "## The results are added to a configuration dictionnary " + ] + }, + { + "cell_type": "markdown", + "id": "3c4e99b3-5d7d-4bc2-8530-d0a70fd8c6be", + "metadata": {}, + "source": [ + "##### Disparities ranges are set" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce0f5e8b-a4c5-4f52-955a-c8bffa49782c", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Disparity range for columns : \", estimation_results[\"input\"][\"col_disparity\"])\n", + "print(\"Disparity range for rows : \", estimation_results[\"input\"][\"row_disparity\"])" + ] + }, + { + "cell_type": "markdown", + "id": "a70b11c1-85bd-450d-86fa-11079a3a7355", + "metadata": {}, + "source": [ + "##### And many informations are stored in estimation dictionnary configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bef0454-425e-43ff-af66-b3d8cd90b31c", + "metadata": {}, + "outputs": [], + "source": [ + "estimation_results[\"pipeline\"][\"estimation\"]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pandora2D-dev", + "language": "python", + "name": "pandora2d-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/img/Pandora2D_pipeline.png b/notebooks/img/Pandora2D_pipeline.png index c4b43ba..f396452 100644 Binary files a/notebooks/img/Pandora2D_pipeline.png and b/notebooks/img/Pandora2D_pipeline.png differ diff --git a/notebooks/introduction_and_basic_usage.ipynb b/notebooks/introduction_and_basic_usage.ipynb index 2348545..aa8636f 100644 --- a/notebooks/introduction_and_basic_usage.ipynb +++ b/notebooks/introduction_and_basic_usage.ipynb @@ -94,7 +94,7 @@ "\n", "* Image pair\n", "* Value associated to no_data images\n", - "* Disparity ranges to explore \n", + "* Disparity ranges to explore (if not estimation step)\n", "* Configuration file\n", "\n", "## Outputs\n", @@ -111,8 +111,9 @@ "## Pandora2D's pipeline\n", "\n", "Pandora2D provides the following steps:\n", - "* matching cost computation (**mandatory**)\n", - "* disparity computation (**mandatory**)\n", + "* estimation computation\n", + "* matching cost computation \n", + "* disparity computation (**mandatory if matching_cost**)\n", "* subpixel disparity refinement" ] }, @@ -131,11 +132,12 @@ "source": [ "### Available implementations for each step\n", "\n", - "| Step | Algorithms implemented |\n", - "|:------------------------------|:--------------------------|\n", - "| Matching cost computation | SAD / SSD / ZNNC |\n", - "| Disparity computation | Winner-Takes-All |\n", - "| Subpixel disparity refinement | Interpolation / Dichotomy |" + "| Step | Algorithms implemented |\n", + "|:------------------------------|:-----------------------------------------|\n", + "| Estimation computation | phase cross correlation |\n", + "| Matching cost computation | SAD / SSD / ZNNC |\n", + "| Disparity computation | Winner-Takes-All |\n", + "| Subpixel disparity refinement | Interpolation / Dichotomy / Optical flow |" ] }, { @@ -344,7 +346,8 @@ " \"invalid_disparity\": -2\n", " },\n", " \"refinement\" : {\n", - " \"refinement_method\" : \"interpolation\"\n", + " \"refinement_method\" : \"optical_flow\",\n", + " \"iterations\": 4\n", " }\n", " }\n", "}" @@ -412,7 +415,7 @@ "metadata": {}, "outputs": [], "source": [ - "dataset = pandora2d.run(\n", + "dataset, _ = pandora2d.run(\n", " pandora2d_machine,\n", " image_datasets.left,\n", " image_datasets.right,\n", @@ -473,6 +476,7 @@ "source": [ "The state machine has three states : \n", "* Begin\n", + "* Assumption\n", "* Cost volumes\n", "* Disparity maps\n", "\n", @@ -542,7 +546,8 @@ " \"invalid_disparity\": -5\n", " },\n", " \"refinement\":{\n", - " \"refinement_method\" : \"interpolation\"\n", + " \"refinement_method\" : \"optical_flow\",\n", + " \"iterations\": 4\n", " }\n", " }\n", "}" @@ -592,7 +597,7 @@ "metadata": {}, "outputs": [], "source": [ - "pandora2d_machine.run_prepare(image_datasets.left, image_datasets.right)" + "pandora2d_machine.run_prepare(image_datasets.left, image_datasets.right, checked_cfg)" ] }, { @@ -770,7 +775,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.16" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/pandora2d/__init__.py b/pandora2d/__init__.py index a8ef1d0..95c03a5 100644 --- a/pandora2d/__init__.py +++ b/pandora2d/__init__.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -23,15 +24,15 @@ This module contains functions to run Pandora pipeline. """ -from typing import Dict, List -import xarray as xr +from typing import Dict +import xarray as xr from pandora import read_config_file, setup_logging from pandora.common import save_config from pandora2d import common from pandora2d.check_configuration import check_conf, check_datasets -from pandora2d.img_tools import get_roi_processing, create_datasets_from_inputs +from pandora2d.img_tools import create_datasets_from_inputs, get_roi_processing from pandora2d.state_machine import Pandora2DMachine @@ -39,7 +40,7 @@ def run( pandora2d_machine: Pandora2DMachine, img_left: xr.Dataset, img_right: xr.Dataset, - cfg_pipeline: Dict[str, dict], + cfg: Dict[str, dict], ): """ Run the Pandora 2D pipeline @@ -56,20 +57,20 @@ def run( - im : 2D (row, col) xarray.DataArray - msk (optional): 2D (row, col) xarray.DataArray :type img_right: xarray.Dataset - :param cfg_pipeline: pipeline configuration - :type cfg_pipeline: Dict[str, dict] + :param cfg: configuration + :type cfg: Dict[str, dict] :return: None """ - pandora2d_machine.run_prepare(img_left, img_right) + pandora2d_machine.run_prepare(img_left, img_right, cfg) - for e in list(cfg_pipeline["pipeline"]): - pandora2d_machine.run(e, cfg_pipeline) + for e in list(cfg["pipeline"]): + pandora2d_machine.run(e, cfg) pandora2d_machine.run_exit() - return pandora2d_machine.dataset_disp_maps + return pandora2d_machine.dataset_disp_maps, pandora2d_machine.completed_cfg def main(cfg_path: str, path_output: str, verbose: bool) -> None: @@ -103,15 +104,18 @@ def main(cfg_path: str, path_output: str, verbose: bool) -> None: roi = get_roi_processing(cfg["ROI"], col_disparity, row_disparity) # read images - image_datasets = create_datasets_from_inputs(input_config=cfg["input"], roi=roi) + image_datasets = create_datasets_from_inputs( + input_config=cfg["input"], roi=roi, estimation_cfg=cfg["pipeline"].get("estimation") + ) # check datasets: shape, format and content check_datasets(image_datasets.left, image_datasets.right) # run pandora 2D and store disp maps in a dataset - dataset_disp_maps = run(pandora2d_machine, image_datasets.left, image_datasets.right, cfg) + dataset_disp_maps, completed_cfg = run(pandora2d_machine, image_datasets.left, image_datasets.right, cfg) - # save dataset - common.save_dataset(dataset_disp_maps, path_output) + # save dataset if not empty + if bool(dataset_disp_maps.data_vars): + common.save_dataset(dataset_disp_maps, path_output) # save config - save_config(path_output, user_cfg) + save_config(path_output, completed_cfg) diff --git a/pandora2d/check_configuration.py b/pandora2d/check_configuration.py index de0c08a..24e65b9 100644 --- a/pandora2d/check_configuration.py +++ b/pandora2d/check_configuration.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -22,14 +22,21 @@ """ This module contains functions allowing to check the configuration given to Pandora pipeline. """ -from typing import Dict -import xarray as xr -from json_checker import Checker, Or, And -import numpy as np -from pandora.check_configuration import check_disparities_from_input, check_images, get_config_input, check_dataset -from pandora.check_configuration import concat_conf, update_conf, rasterio_can_open_mandatory +from typing import Dict +import numpy as np +import xarray as xr +from json_checker import And, Checker, Or +from pandora.check_configuration import ( + check_dataset, + check_disparities_from_input, + check_images, + concat_conf, + get_config_input, + rasterio_can_open_mandatory, + update_conf, +) from pandora2d.state_machine import Pandora2DMachine @@ -58,12 +65,14 @@ def check_datasets(left: xr.Dataset, right: xr.Dataset) -> None: raise ValueError("left and right datasets must have the same shape") -def check_input_section(user_cfg: Dict[str, dict]) -> Dict[str, dict]: +def check_input_section(user_cfg: Dict[str, dict], estimation_config: dict = None) -> Dict[str, dict]: """ Complete and check if the dictionary is correct :param user_cfg: user configuration :type user_cfg: dict + :param estimation_config: get estimation config if in user_config + :type estimation_config: dict :return: cfg: global configuration :rtype: cfg: dict """ @@ -71,6 +80,14 @@ def check_input_section(user_cfg: Dict[str, dict]) -> Dict[str, dict]: # Add missing steps and inputs defaults values in user_cfg cfg = update_conf(default_short_configuration_input, user_cfg) + # test disparities + if estimation_config is None: + check_disparities_from_input(cfg["input"]["col_disparity"], None) + check_disparities_from_input(cfg["input"]["row_disparity"], None) + else: + # add wrong disparity for checking only + cfg = update_conf(default_configuration_disp, user_cfg) + # check schema configuration_schema = {"input": input_configuration_schema} checker = Checker(configuration_schema) @@ -79,10 +96,6 @@ def check_input_section(user_cfg: Dict[str, dict]) -> Dict[str, dict]: # test images check_images(cfg["input"]) - # test disparities - check_disparities_from_input(cfg["input"]["col_disparity"], None) - check_disparities_from_input(cfg["input"]["row_disparity"], None) - return cfg @@ -158,7 +171,7 @@ def check_conf(user_cfg: Dict, pandora2d_machine: Pandora2DMachine) -> dict: # check input user_cfg_input = get_config_input(user_cfg) - cfg_input = check_input_section(user_cfg_input) + cfg_input = check_input_section(user_cfg_input, user_cfg["pipeline"].get("estimation")) user_cfg_roi = get_roi_config(user_cfg) cfg_roi = check_roi_section(user_cfg_roi) @@ -166,7 +179,9 @@ def check_conf(user_cfg: Dict, pandora2d_machine: Pandora2DMachine) -> dict: # check pipeline cfg_pipeline = check_pipeline_section(user_cfg, pandora2d_machine) - check_right_nodata_condition(cfg_input, cfg_pipeline) + # The estimation step can be utilized independently. + if "matching_cost" in cfg_pipeline["pipeline"]: + check_right_nodata_condition(cfg_input, cfg_pipeline) cfg = concat_conf([cfg_input, cfg_roi, cfg_pipeline]) @@ -181,6 +196,7 @@ def check_right_nodata_condition(cfg_input: Dict, cfg_pipeline: Dict) -> None: :param cfg_pipeline: pipeline section of configuration :type cfg_pipeline: Dict """ + if not isinstance(cfg_input["input"]["right"]["nodata"], int) and cfg_pipeline["pipeline"]["matching_cost"][ "matching_cost_method" ] in ["sad", "ssd"]: @@ -244,6 +260,8 @@ def get_roi_config(user_cfg: Dict[str, dict]) -> Dict[str, dict]: } } +default_configuration_disp = {"input": {"col_disparity": [-9999, -9999], "row_disparity": [-9999, -9999]}} + roi_configuration_schema = { "row": {"first": And(int, lambda x: x >= 0), "last": And(int, lambda x: x >= 0)}, "col": {"first": And(int, lambda x: x >= 0), "last": And(int, lambda x: x >= 0)}, diff --git a/pandora2d/common.py b/pandora2d/common.py index 2f9e645..f2a9a77 100644 --- a/pandora2d/common.py +++ b/pandora2d/common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -53,7 +53,7 @@ def save_dataset(dataset: xr.Dataset, output: str) -> None: write_data_array(dataset["col_map"], os.path.join(output, "columns_disparity.tif")) -def dataset_disp_maps(delta_row: np.ndarray, delta_col: np.ndarray) -> xr.Dataset: +def dataset_disp_maps(delta_row: np.ndarray, delta_col: np.ndarray, attributes: dict = None) -> xr.Dataset: """ Create the dataset containing disparity maps @@ -61,6 +61,8 @@ def dataset_disp_maps(delta_row: np.ndarray, delta_col: np.ndarray) -> xr.Datase :type delta_row: np.ndarray :param delta_col: disparity map for col :type delta_col: np.ndarray + :param attributes: attributes containing invalid disparity values + :type attributes: dict :return: dataset: Dataset with the disparity maps with the data variables : - row_map 2D xarray.DataArray (row, col) @@ -81,4 +83,7 @@ def dataset_disp_maps(delta_row: np.ndarray, delta_col: np.ndarray) -> xr.Datase # merge two dataset into one dataset = dataset_row.merge(dataset_col, join="override", compat="override") + if attributes is not None: + dataset.attrs = attributes + return dataset diff --git a/pandora2d/estimation/__init__.py b/pandora2d/estimation/__init__.py new file mode 100644 index 0000000..8846ad9 --- /dev/null +++ b/pandora2d/estimation/__init__.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2024 CS GROUP France +# +# This file is part of PANDORA2D +# +# https://github.com/CNES/Pandora2D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Init file for estimation module +""" + +from . import phase_cross_correlation +from .estimation import AbstractEstimation diff --git a/pandora2d/estimation/estimation.py b/pandora2d/estimation/estimation.py new file mode 100644 index 0000000..d7a933d --- /dev/null +++ b/pandora2d/estimation/estimation.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# +# Copyright (c) 2024 CS GROUP France +# +# This file is part of PANDORA2D +# +# https://github.com/CNES/Pandora2D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module contains functions associated to the estimation computation step. +""" +from __future__ import annotations + +import logging +from abc import ABCMeta, abstractmethod +from typing import Dict, Tuple + +import numpy as np +import xarray as xr + + +class AbstractEstimation: + """ + Abstract Estimation class + """ + + __metaclass__ = ABCMeta + + estimation_methods_avail: Dict = {} + _estimation_method = None + cfg = None + + def __new__(cls, cfg: dict | None = None): + """ + Return the plugin associated with the estimation_method given in the configuration + + :param cfg: configuration {'estimation_method': value} + :type cfg: dictionary + """ + if cls is AbstractEstimation: + if isinstance(cfg["estimation_method"], str): + try: + return super(AbstractEstimation, cls).__new__( + cls.estimation_methods_avail[cfg["estimation_method"]] + ) + except KeyError: + logging.error("No estimation method named % supported", cfg["estimation_method"]) + raise KeyError + else: + if isinstance(cfg["estimation_method"], unicode): # type: ignore # pylint: disable=undefined-variable + # creating a plugin from registered short name given as unicode (py2 & 3 compatibility) + try: + return super(AbstractEstimation, cls).__new__( + cls.estimation_methods_avail[cfg["estimation_method"].encode("utf-8")] + ) + except KeyError: + logging.error( + "No estimation method named % supported", + cfg["estimation_method"], + ) + raise KeyError + else: + return super(AbstractEstimation, cls).__new__(cls) + return None + + @classmethod + def register_subclass(cls, short_name: str): + """ + Allows to register the subclass with its short name + + :param short_name: the subclass to be registered + :type short_name: string + """ + + def decorator(subclass): + """ + Registers the subclass in the available methods + + :param subclass: the subclass to be registered + :type subclass: object + """ + cls.estimation_methods_avail[short_name] = subclass + return subclass + + return decorator + + def desc(self) -> None: + """ + Describes the estimation method + :return: None + """ + print(f"{self._estimation_method} estimation measure") + + @abstractmethod + def compute_estimation(self, img_left: xr.Dataset, img_right: xr.Dataset) -> Tuple[list, list, np.ndarray, dict]: + """ + Compute the phase cross correlation method + + :param img_left: xarray.Dataset containing : + - im : 2D (row, col) xarray.DataArray + :type img_left: xr.Dataset + :param img_right: xarray.Dataset containing : + - im : 2D (row, col) xarray.DataArray + :type img_right: xr.Dataset + :return:row disparity: list + col disparity: list + Calculated shifts: list + Extra information about estimation : dict + :rtype: list, list, np.ndarray, dict + """ + + @staticmethod + def update_cfg_with_estimation( + cfg: Dict, disp_col: list, disp_row: list, shifts: np.ndarray, extra_dict: dict = None + ) -> Dict: + """ + Save calculated shifts in a configuration dictionary + + :param cfg: user configuration + :type cfg: dict + :param disp_col: list of min and max disparity in column + :type disp_col: [int, int] + :param disp_row: list of min and max disparity in row + :type disp_row: [int, int] + :param shifts: computed global shifts between left and right + :type shifts: [np.float32, np.float32] + :param extra_dict: Dictionary containing extra information about estimation + :type extra_dict: dict + :return: cfg: global configuration + :rtype: cfg: dict + """ + + cfg["input"]["col_disparity"] = disp_col + cfg["input"]["row_disparity"] = disp_row + + cfg["pipeline"]["estimation"]["estimated_shifts"] = shifts.tolist() + if extra_dict is not None: + for key, value in extra_dict.items(): + cfg["pipeline"]["estimation"][key] = value + + return cfg diff --git a/pandora2d/estimation/phase_cross_correlation.py b/pandora2d/estimation/phase_cross_correlation.py new file mode 100644 index 0000000..903a2d0 --- /dev/null +++ b/pandora2d/estimation/phase_cross_correlation.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# +# Copyright (c) 2024 CS GROUP France +# +# This file is part of PANDORA2D +# +# https://github.com/CNES/Pandora2D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module contains functions associated to the phase cross correlation method used in the estimation step. +""" + +import logging +from typing import Dict, Tuple + +import numpy as np +import xarray as xr +from json_checker import And, Checker +from skimage.registration import phase_cross_correlation + +from . import estimation + + +@estimation.AbstractEstimation.register_subclass("phase_cross_correlation") +class PhaseCrossCorrelation(estimation.AbstractEstimation): + """ + PhaseCrossCorrelation class allows to perform estimation + """ + + _range_col = None + _range_row = None + _sample_factor = None + + # Default configuration, do not change these values + _RANGE_COL = 5 + _RANGE_ROW = 5 + _SAMPLE_FACTOR = 1 + + def __init__(self, cfg: Dict) -> None: + """ + :param cfg: optional configuration, {'range_col': int, 'range_row': int, 'sample_factor': int} + :type cfg: dict + :return: None + """ + + self.cfg = self.check_conf(cfg) + self._range_col = self.cfg["range_col"] + self._range_row = self.cfg["range_row"] + self._sample_factor = self.cfg["sample_factor"] + self._estimation_method = self.cfg["estimation_method"] + + def check_conf(self, cfg: Dict) -> Dict: + """ + Check the estimation configuration + + :param cfg: user_config for refinement + :type cfg: dict + :return: cfg: global configuration + :rtype: cfg: dict + """ + + # Give the default value if the required element is not in the conf + if "range_col" not in cfg: + cfg["range_col"] = self._RANGE_COL + + if "range_row" not in cfg: + cfg["range_row"] = self._RANGE_ROW + + if "sample_factor" not in cfg: + cfg["sample_factor"] = self._SAMPLE_FACTOR + + # Estimation schema config + schema = { + "estimation_method": And(str, lambda estimation_method: estimation_method in ["phase_cross_correlation"]), + "range_col": And(int, lambda range_col: range_col > 2), + "range_row": And(int, lambda range_row: range_row > 2), + "sample_factor": And(int, lambda sf: sf % 10 == 0 or sf == 1, lambda sf: sf > 0), + } + + checker = Checker(schema) + checker.validate(cfg) + + return cfg + + def compute_estimation(self, img_left: xr.Dataset, img_right: xr.Dataset) -> Tuple[list, list, np.ndarray, dict]: + """ + Compute the phase cross correlation method + + :param img_left: xarray.Dataset containing : + - im : 2D (row, col) xarray.DataArray + :type img_left: xr.Dataset + :param img_right: xarray.Dataset containing : + - im : 2D (row, col) xarray.DataArray + :type img_right: xr.Dataset + :return: row disparity: list + col disparity: list + Calculated shifts: np.ndarray + Extra information about estimation: dict + :rtype: list, list, np.ndarray, dict + """ + + # https://scikit-image.org/docs/stable/api/ + # skimage.registration.html#skimage.registration.phase_cross_correlation + shifts, error, phasediff = phase_cross_correlation( + img_left["im"].data, img_right["im"].data, upsample_factor=self._sample_factor + ) + + # reformat outputs + phasediff = "{:.{}e}".format(phasediff, 8) + # -shifts because of pandora2d convention + min_col = round(-shifts[1]) - int(self._range_col) + max_col = round(-shifts[1]) + int(self._range_col) + + min_row = round(-shifts[0]) - int(self._range_row) + max_row = round(-shifts[0]) + int(self._range_row) + + row_disparity = [min_row, max_row] + col_disparity = [min_col, max_col] + + logging.info("Estimation result is %s in columns and %s in row", -shifts[1], -shifts[0]) + logging.debug("Translation invariant normalized RMS error between left and right is %s", error) + logging.debug("Global phase difference between the two images is %s", phasediff) + + extra_dict = {"error": error, "phase_diff": phasediff} + + return row_disparity, col_disparity, -shifts, extra_dict diff --git a/pandora2d/img_tools.py b/pandora2d/img_tools.py index 7059b46..8b41e3b 100644 --- a/pandora2d/img_tools.py +++ b/pandora2d/img_tools.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -42,7 +42,7 @@ class Datasets(NamedTuple): right: xr.Dataset -def create_datasets_from_inputs(input_config: Dict, roi: Dict = None) -> Datasets: +def create_datasets_from_inputs(input_config: Dict, roi: Dict = None, estimation_cfg: Dict = None) -> Datasets: """ Read image and return the corresponding xarray.DataSet @@ -56,6 +56,8 @@ def create_datasets_from_inputs(input_config: Dict, roi: Dict = None) -> Dataset with margins : left, up, right, down :type roi: dict + :param estimation_cfg: dictionary containing estimation configuration + :type estimation_cfg: dict :return: Datasets NamedTuple with two attributes `left` and `right` each containing a xarray.DataSet containing the variables : @@ -66,7 +68,12 @@ def create_datasets_from_inputs(input_config: Dict, roi: Dict = None) -> Dataset :rtype: Datasets """ - check_disparities(input_config) + if estimation_cfg is None: + check_disparities(input_config) + else: + input_config["col_disparity"] = [-9999, -9999] + input_config["row_disparity"] = [-9999, -9999] + return Datasets( pandora_img_tools.create_dataset_from_inputs(input_config["left"], roi).pipe( add_left_disparity_grid, input_config diff --git a/pandora2d/matching_cost/matching_cost.py b/pandora2d/matching_cost/matching_cost.py index 7df6f59..28b2a3e 100644 --- a/pandora2d/matching_cost/matching_cost.py +++ b/pandora2d/matching_cost/matching_cost.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -252,5 +253,6 @@ def compute_cost_volumes( del cost_volumes.attrs["disparity_source"] cost_volumes.attrs["col_disparity_source"] = img_left.attrs["col_disparity_source"] cost_volumes.attrs["row_disparity_source"] = img_left.attrs["row_disparity_source"] + cost_volumes.attrs["step"] = self.cfg["step"] return cost_volumes diff --git a/pandora2d/refinement/__init__.py b/pandora2d/refinement/__init__.py index 21a5a84..1f58be2 100644 --- a/pandora2d/refinement/__init__.py +++ b/pandora2d/refinement/__init__.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -23,6 +22,6 @@ Init file for refinement module """ -from . import interpolation +from . import interpolation, optical_flow from . import dichotomy from .refinement import AbstractRefinement diff --git a/pandora2d/refinement/dichotomy.py b/pandora2d/refinement/dichotomy.py index 3e972f2..b36e044 100644 --- a/pandora2d/refinement/dichotomy.py +++ b/pandora2d/refinement/dichotomy.py @@ -1,4 +1,5 @@ # Copyright (c) 2024 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -66,14 +67,20 @@ def check_conf(cls, cfg: Dict) -> Dict: def margins(self): return Margins(2, 2, 2, 2) - def refinement_method(self, cost_volumes: xr.Dataset, pixel_maps: xr.Dataset) -> None: + def refinement_method( + self, cost_volumes: xr.Dataset, disp_map: xr.Dataset, img_left: xr.Dataset, img_right: xr.Dataset + ) -> None: """ Return the subpixel disparity maps :param cost_volumes: cost_volumes 4D row, col, disp_col, disp_row - :type cost_volumes: xarray.dataset - :param pixel_maps: pixels disparity maps - :type pixel_maps: xarray.dataset + :type cost_volumes: xarray.Dataset + :param disp_map: pixel disparity maps + :type disp_map: xarray.Dataset + :param img_left: left image dataset + :type img_left: xarray.Dataset + :param img_right: right image dataset + :type img_right: xarray.Dataset :return: the refined disparity maps :rtype: Tuple[np.ndarray, np.ndarray] """ diff --git a/pandora2d/refinement/interpolation.py b/pandora2d/refinement/interpolation.py index c6efcc7..cdd0f07 100644 --- a/pandora2d/refinement/interpolation.py +++ b/pandora2d/refinement/interpolation.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -126,13 +126,19 @@ def compute_cost_matrix(self, p_args) -> Tuple[float, float]: return res - def refinement_method(self, cost_volumes: xr.Dataset, pixel_maps: xr.Dataset) -> Tuple[np.ndarray, np.ndarray]: + def refinement_method( + self, cost_volumes: xr.Dataset, disp_map: xr.Dataset, img_left: xr.Dataset, img_right: xr.Dataset + ) -> Tuple[np.ndarray, np.ndarray]: """ Compute refine disparity maps :param cost_volumes: Cost_volumes has (row, col, disp_col, disp_row) dimensions :type cost_volumes: xr.Dataset - :param pixel_maps: dataset of pixel disparity maps - :type pixel_maps: xr.Dataset + :param disp_map: dataset of pixel disparity maps + :type disp_map: xr.Dataset + :param img_left: left image dataset + :type img_left: xarray.Dataset + :param img_right: right image dataset + :type img_right: xarray.Dataset :return: delta_col, delta_row: subpixel disparity maps :rtype: Tuple[np.array, np.array] """ @@ -144,8 +150,8 @@ def refinement_method(self, cost_volumes: xr.Dataset, pixel_maps: xr.Dataset) -> cost_matrix = np.rollaxis(np.rollaxis(data, 3, 0), 3, 1).reshape((ndisprow, ndispcol, nrow * ncol)) # flatten pixel maps for multiprocessing - liste_row = list(pixel_maps["row_map"].data.flatten().tolist()) - liste_col = list(pixel_maps["col_map"].data.flatten().tolist()) + liste_row = list(disp_map["row_map"].data.flatten().tolist()) + liste_col = list(disp_map["col_map"].data.flatten().tolist()) # args for multiprocessing args = [ @@ -160,7 +166,7 @@ def refinement_method(self, cost_volumes: xr.Dataset, pixel_maps: xr.Dataset) -> delta_row = np.array(map_carte)[:, 1] # reshape disparity maps - delta_col = np.reshape(delta_col, (pixel_maps["col_map"].data.shape[0], pixel_maps["col_map"].data.shape[1])) - delta_row = np.reshape(delta_row, (pixel_maps["col_map"].data.shape[0], pixel_maps["col_map"].data.shape[1])) + delta_col = np.reshape(delta_col, (disp_map["col_map"].data.shape[0], disp_map["col_map"].data.shape[1])) + delta_row = np.reshape(delta_row, (disp_map["col_map"].data.shape[0], disp_map["col_map"].data.shape[1])) return delta_col, delta_row diff --git a/pandora2d/refinement/optical_flow.py b/pandora2d/refinement/optical_flow.py new file mode 100644 index 0000000..3555512 --- /dev/null +++ b/pandora2d/refinement/optical_flow.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python +# +# Copyright (c) 2024 CS GROUP France +# +# This file is part of PANDORA2D +# +# https://github.com/CNES/Pandora2D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +This module contains functions associated to the optical flow method used in the refinement step. +""" +from typing import Dict, Tuple + +import numpy as np +import xarray as xr +from json_checker import And +from scipy.ndimage import map_coordinates +from pandora.margins import Margins + +from . import refinement + + +@refinement.AbstractRefinement.register_subclass("optical_flow") +class OpticalFlow(refinement.AbstractRefinement): + """ + OpticalFLow class allows to perform the subpixel cost refinement step + """ + + _iterations = None + _invalid_disp = None + + _ITERATIONS = 4 + + schema = {"refinement_method": And(str, lambda x: x in ["optical_flow"]), "iterations": And(int, lambda it: it > 0)} + + def __init__(self, cfg: dict = None, step: list = None, window_size: int = 5) -> None: + """ + :param cfg: optional configuration, {} + :type cfg: dict + :param step: list containing row and col step + :type step: list + :param window_size: window size + :type window_size: int + :return: None + """ + super().__init__(cfg) + + self._iterations = self.cfg["iterations"] + self._refinement_method = self.cfg["refinement_method"] + self._window_size = window_size + self._step = [1, 1] if step is None else step + + @classmethod + def check_conf(cls, cfg: Dict) -> Dict: + """ + Check the refinement configuration + + :param cfg: user_config for refinement + :type cfg: dict + :return: cfg: global configuration + :rtype: cfg: dict + """ + + cfg["iterations"] = cfg.get("iterations", cls._ITERATIONS) + + cfg = super().check_conf(cfg) + + return cfg + + @property + def margins(self): + values = (self._window_size // 2 * ele for _ in range(2) for ele in self._step) + return Margins(*values) + + def reshape_to_matching_cost_window( + self, + img: xr.Dataset, + cost_volumes: xr.Dataset, + disp_row: np.ndarray = None, + disp_col: np.ndarray = None, + ): + """ + Transform image from (nb_col, nb_row) to (window_size, window_size, nbcol*nbrow) + + :param img: image to reshape + :type img: xr.Dataset + :param cost_volumes: cost_volumes 4D row, col, disp_col, disp_row + :type cost_volumes: xarray.Dataset + :param disp_row: array dim [] containing all the row shift + :type disp_row: np.ndarray + :param disp_col: array dim [] containing all the columns shift + :type disp_col: np.ndarray + :return: array containing reshaped image [window_size, window_size, nbcol*nbrow] + :rtype: np.ndarray + """ + + # get numpy array datas for image + img_data = img["im"].data + + offset = max(self.margins.astuple()) + + computable_col = cost_volumes.col.data[offset:-offset] + computable_row = cost_volumes.row.data[offset:-offset] + + one_dim_size = len(computable_row) * len(computable_col) + + if disp_row is None and disp_col is None: + patches = np.lib.stride_tricks.sliding_window_view(img_data, [self._window_size, self._window_size]) + patches = patches.reshape((one_dim_size, self._window_size, self._window_size)).transpose((1, 2, 0)) + else: + # initiate values for right reshape computation + offset = max(self.margins.astuple()) + patches = np.ndarray((self._window_size, self._window_size, one_dim_size)) + idx = 0 + + for row in computable_row: + for col in computable_col: + + shift_col = ( + 0 if np.isnan(disp_col[idx]) or disp_col[idx] == self._invalid_disp else int(disp_col[idx]) + ) + shift_row = ( + 0 if np.isnan(disp_row[idx]) or disp_row[idx] == self._invalid_disp else int(disp_row[idx]) + ) + + # get right pixel with his matching cost window + patch = img_data[ + row - offset + shift_row : row + offset + 1 + shift_row, + col - offset + shift_col : col + offset + 1 + shift_col, + ] + + # stock matching_cost window + if patch.shape == (self._window_size, self._window_size): + patches[:, :, idx] = patch + else: + patches[:, :, idx] = np.ones([self._window_size, self._window_size]) * np.nan + + idx += 1 + + return patches + + def warped_img( + self, right_reshape: np.ndarray, delta_row: np.ndarray, delta_col: np.ndarray, index_to_compute: list + ): + """ + Shifted matching_cost window with computed disparity + + :param right_reshape: image right reshaped with dims (window_size, window_size, nbcol*nb_row) + :type right_reshape: np.ndarray + :param delta_row: rows disparity map + :type delta_row: np.ndarray + :param delta_col: columns disparity map + :type delta_col: np.ndarray + :param index_to_compute: list containing all valid pixel for computing optical flow + :type index_to_compute: list + :return: new array containing shifted matching_cost windows + :rtype: np.ndarray + """ + + x, y = np.meshgrid(range(self._window_size), range(self._window_size)) + + new_img = np.empty_like(right_reshape) + + # resample matching cost right windows + for idx in index_to_compute: + shifted_img = map_coordinates( + right_reshape[:, :, idx], [y - delta_row[idx], x - delta_col[idx]], order=5, mode="reflect" + ) + + new_img[:, :, idx] = shifted_img + + return new_img + + def lucas_kanade_core_algorithm(self, left_data: np.ndarray, right_data: np.ndarray) -> Tuple[float, float]: + """ + Implement lucas & kanade algorithm core + + :param left_data: matching_cost window for one pixel from left image + :type left_data: np.ndarray + :param right_data: matching_cost window for one pixel from left image + :type right_data: np.ndarray + :return: sub-pixel disparity computed by Lucas & Kanade optical flow + :rtype: Tuple[float, float] + """ + + grad_y, grad_x = np.gradient(left_data) + grad_t = right_data - left_data + + # Create A (grad_matrix) et B (time_matrix) matrix for Lucas Kanade + grad_matrix = np.vstack((grad_x.flatten(), grad_y.flatten())).T + time_matrix = grad_t.flatten() + + # Apply least-squares to solve the matrix equation AV= B where A is matrix containing partial derivate of (x,y) + # B the matrix of partial derivate of t and V the motion we want to find + + try: + motion = np.linalg.lstsq(grad_matrix, time_matrix, rcond=None)[0] + # if matrix is full of NaN or 0 + except np.linalg.LinAlgError: + motion = (self._invalid_disp, self._invalid_disp) + + return motion[1], motion[0] + + def optical_flow( + self, + left_img: np.ndarray, + right_img: np.ndarray, + list_idx_to_compute: list, + ) -> Tuple[np.ndarray, np.ndarray, list]: + """ + Computing optical flow between left and right image + + :param left_img: reshaped left image array + :type left_img: np.ndarray + :param right_img: reshaped right image array + :type right_img: np.ndarray + :param list_idx_to_compute: list of valid pixel + :type list_idx_to_compute: list + :return: computed sub-pixel disparity map + :rtype: Tuple[np.ndarray, np.ndarray, list] + """ + + new_list_to_compute = [] + + final_dec_row = np.zeros(left_img.shape[2]) + final_dec_col = np.zeros(left_img.shape[2]) + + for idx in list_idx_to_compute: + + left_matching_cost = left_img[:, :, idx] + right_matching_cost = right_img[:, :, idx] + + computed_delta_row, computed_delta_col = self.lucas_kanade_core_algorithm( + left_matching_cost, right_matching_cost + ) + + # hypothesis from algorithm: shifts are < 1 + if abs(computed_delta_col) < 1 and abs(computed_delta_row) < 1: + new_list_to_compute.append(idx) + else: + if abs(computed_delta_col) > 1: + computed_delta_col = 0 + if abs(computed_delta_row) > 1: + computed_delta_row = 0 + + final_dec_row[idx] = computed_delta_row + final_dec_col[idx] = computed_delta_col + + return final_dec_row, final_dec_col, new_list_to_compute + + def refinement_method( + self, cost_volumes: xr.Dataset, disp_map: xr.Dataset, img_left: xr.Dataset, img_right: xr.Dataset + ) -> Tuple[np.ndarray, np.ndarray]: + """ + Return the subpixel disparity maps + + :param cost_volumes: cost_volumes 4D row, col, disp_col, disp_row + :type cost_volumes: xarray.Dataset + :param disp_map: pixels disparity maps + :type disp_map: xarray.Dataset + :param img_left: left image dataset + :type img_left: xarray.Dataset + :param img_right: right image dataset + :type img_right: xarray.Dataset + :return: the refined disparity maps + :rtype: Tuple[np.ndarray, np.ndarray] + """ + + # get invalid_disp value + self._invalid_disp = disp_map.attrs["invalid_disp"] + + # get offset + offset = max(self.margins.astuple()) + + # get displacement map from disparity state + initial_delta_row = disp_map["row_map"].data + initial_delta_col = disp_map["col_map"].data + + delta_col = initial_delta_col[offset:-offset, offset:-offset].flatten() + delta_row = initial_delta_row[offset:-offset, offset:-offset].flatten() + + # reshape left and right datas + # from (nbcol, nbrow) to (window_size, window_size, nbcol*nbrow) + reshaped_left = self.reshape_to_matching_cost_window(img_left, cost_volumes) + reshaped_right = self.reshape_to_matching_cost_window( + img_right, + cost_volumes, + delta_row, + delta_col, + ) + + idx_to_compute = np.arange(reshaped_left.shape[2]).tolist() + + for _ in range(self._iterations): + + computed_drow, computed_dcol, idx_to_compute = self.optical_flow( + reshaped_left, reshaped_right, idx_to_compute + ) + + reshaped_right = self.warped_img(reshaped_right, computed_drow, computed_dcol, idx_to_compute) + + # Pandora convention is left - d = right + # Lucas&Kanade convention is left + d = right + delta_col = delta_col - computed_dcol + delta_row = delta_row - computed_drow + + # get finals disparity map dimensions + nb_row, nb_col = initial_delta_col.shape + nb_valid_points_row = nb_row - 2 * offset + nb_valid_points_col = nb_col - 2 * offset + + delta_col = delta_col.reshape([nb_valid_points_col, nb_valid_points_row]) + delta_row = delta_row.reshape([nb_valid_points_col, nb_valid_points_row]) + + # add borders + delta_col = np.pad(delta_col, pad_width=offset, constant_values=self._invalid_disp) + delta_row = np.pad(delta_row, pad_width=offset, constant_values=self._invalid_disp) + + delta_col[delta_col <= img_left.attrs["row_disparity_source"][0]] = self._invalid_disp + delta_col[delta_col >= img_left.attrs["row_disparity_source"][1]] = self._invalid_disp + delta_row[delta_row <= img_left.attrs["col_disparity_source"][0]] = self._invalid_disp + delta_row[delta_row >= img_left.attrs["col_disparity_source"][1]] = self._invalid_disp + + return delta_col, delta_row diff --git a/pandora2d/refinement/refinement.py b/pandora2d/refinement/refinement.py index e86f020..876a74a 100644 --- a/pandora2d/refinement/refinement.py +++ b/pandora2d/refinement/refinement.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -50,13 +50,14 @@ class AbstractRefinement: # If we don't make cfg optional, we got this error when we use subprocesses in refinement_method : # AbstractRefinement.__new__() missing 1 required positional argument: 'cfg' - def __new__(cls, cfg: dict | None = None): + def __new__(cls, cfg: dict = None, _: list = None, __: int = 5): """ Return the plugin associated with the refinement_method given in the configuration :param cfg: configuration {'refinement_method': value} :type cfg: dictionary """ + if cls is AbstractRefinement: if isinstance(cfg["refinement_method"], str): try: @@ -83,6 +84,13 @@ def __new__(cls, cfg: dict | None = None): return super(AbstractRefinement, cls).__new__(cls) return None + def desc(self) -> None: + """ + Describes the refinement method + :return: None + """ + print(f"{self._refinement_method} refinement measure") + @classmethod def register_subclass(cls, short_name: str): """ @@ -104,10 +112,14 @@ def decorator(subclass): return decorator - def __init__(self, cfg: Dict) -> None: + def __init__(self, cfg: Dict, _: list = None, __: int = 5) -> None: """ :param cfg: optional configuration, {} :type cfg: dict + :param step: list containing row and col step + :type step: list + :param window_size: window size + :type window_size: int :return: None """ self.cfg = self.check_conf(cfg) @@ -128,14 +140,20 @@ def check_conf(cls, cfg: Dict) -> Dict: return cfg @abstractmethod - def refinement_method(self, cost_volumes: xr.Dataset, pixel_maps: xr.Dataset) -> Tuple[np.ndarray, np.ndarray]: + def refinement_method( + self, cost_volumes: xr.Dataset, disp_map: xr.Dataset, img_left: xr.Dataset, img_right: xr.Dataset + ) -> Tuple[np.ndarray, np.ndarray]: """ Return the subpixel disparity maps :param cost_volumes: cost_volumes 4D row, col, disp_col, disp_row - :type cost_volumes: xarray.dataset - :param pixel_maps: pixels disparity maps - :type pixel_maps: xarray.dataset + :type cost_volumes: xarray.Dataset + :param disp_map: pixels disparity maps + :type disp_map: xarray.Dataset + :param img_left: left image dataset + :type img_left: xarray.Dataset + :param img_right: right image dataset + :type img_right: xarray.Dataset :return: the refined disparity maps :rtype: Tuple[np.ndarray, np.ndarray] """ diff --git a/pandora2d/state_machine.py b/pandora2d/state_machine.py index 36fac29..e4d80c4 100644 --- a/pandora2d/state_machine.py +++ b/pandora2d/state_machine.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -22,10 +22,11 @@ """ This module contains class associated to the pandora state machine """ - -from typing import Dict, TYPE_CHECKING, List, TypedDict, Literal, Optional, Union +import copy import logging from operator import add +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, TypedDict, Union + import numpy as np import xarray as xr from typing_extensions import Annotated @@ -42,10 +43,11 @@ from transitions.extensions import GraphMachine as Machine except ImportError: from transitions import Machine + from transitions import MachineError from pandora.margins import GlobalMargins -from pandora2d import matching_cost, disparity, refinement, common +from pandora2d import common, disparity, estimation, matching_cost, refinement, img_tools class MarginsProperties(TypedDict): @@ -55,12 +57,13 @@ class MarginsProperties(TypedDict): margins: Annotated[List[int], '["left, "up", "right", "down"]'] -class Pandora2DMachine(Machine): +class Pandora2DMachine(Machine): # pylint:disable=too-many-instance-attributes """ - Pandora2DMacine class to create and use a state machine + Pandora2DMachine class to create and use a state machine """ _transitions_run = [ + {"trigger": "estimation", "source": "begin", "dest": "assumption", "after": "estimation_run"}, { "trigger": "matching_cost", "source": "begin", @@ -68,12 +71,26 @@ class Pandora2DMachine(Machine): "prepare": "matching_cost_prepare", "after": "matching_cost_run", }, + { + "trigger": "matching_cost", + "source": "assumption", + "dest": "cost_volumes", + "prepare": "matching_cost_prepare", + "after": "matching_cost_run", + }, {"trigger": "disparity", "source": "cost_volumes", "dest": "disp_maps", "after": "disp_maps_run"}, {"trigger": "refinement", "source": "disp_maps", "dest": "disp_maps", "after": "refinement_run"}, ] _transitions_check = [ + {"trigger": "estimation", "source": "begin", "dest": "assumption", "after": "estimation_check_conf"}, {"trigger": "matching_cost", "source": "begin", "dest": "cost_volumes", "after": "matching_cost_check_conf"}, + { + "trigger": "matching_cost", + "source": "assumption", + "dest": "cost_volumes", + "after": "matching_cost_check_conf", + }, {"trigger": "disparity", "source": "cost_volumes", "dest": "disp_maps", "after": "disparity_check_conf"}, {"trigger": "refinement", "source": "disp_maps", "dest": "disp_maps", "after": "refinement_check_conf"}, ] @@ -100,11 +117,16 @@ def __init__( self.disp_max_row: np.ndarray = None self.pipeline_cfg: Dict = {"pipeline": {}} + self.completed_cfg: Dict = {} self.cost_volumes: xr.Dataset = xr.Dataset() self.dataset_disp_maps: xr.Dataset = xr.Dataset() - # Define avalaible states - states_ = ["begin", "cost_volumes", "disp_maps"] + # For communication between matching_cost and refinement steps + self.step: list = None + self.window_size: int = None + + # Define available states + states_ = ["begin", "assumption", "cost_volumes", "disp_maps"] # Instance matching_cost self.matching_cost_: Union[matching_cost.MatchingCost, None] = None @@ -120,11 +142,7 @@ def __init__( logging.getLogger("transitions").setLevel(logging.WARNING) - def run_prepare( - self, - img_left: xr.Dataset, - img_right: xr.Dataset, - ) -> None: + def run_prepare(self, img_left: xr.Dataset, img_right: xr.Dataset, cfg: dict) -> None: """ Prepare the machine before running @@ -138,6 +156,8 @@ def run_prepare( - im : 2D (row, col) xarray.DataArray - msk : 2D (row, col) xarray.DataArray :type img_right: xarray.Dataset + :param cfg: configuration + :type cfg: Dict[str, dict] """ self.left_img = img_left @@ -148,6 +168,7 @@ def run_prepare( # Row's min, max disparities self.disp_min_row = img_left["row_disparity"].sel(band_disp="min").data self.disp_max_row = img_left["row_disparity"].sel(band_disp="max").data + self.completed_cfg = copy.copy(cfg) self.add_transitions(self._transitions_run) @@ -239,6 +260,20 @@ def get_global_margins(self): return [max(x, y) for x, y in zip(max_margins, aggregate_margins)] + def estimation_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None: + """ + Check the estimation computation configuration + + :param cfg: configuration + :type cfg: dict + :param input_step: current step + :type input_step: string + :return: None + """ + + estimation_ = estimation.AbstractEstimation(cfg["pipeline"][input_step]) # type: ignore[abstract] + self.pipeline_cfg["pipeline"][input_step] = estimation_.cfg + def matching_cost_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None: """ Check the disparity computation configuration @@ -252,6 +287,8 @@ def matching_cost_check_conf(self, cfg: Dict[str, dict], input_step: str) -> Non matching_cost_ = matching_cost.MatchingCost(cfg["pipeline"][input_step]) self.pipeline_cfg["pipeline"][input_step] = matching_cost_.cfg + self.step = list(matching_cost_.cfg["step"]) + self.window_size = int(matching_cost_.cfg["window_size"]) self.margins.add_cumulative(input_step, matching_cost_.margins) def disparity_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None: @@ -280,7 +317,9 @@ def refinement_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None: :return: None """ - refinement_ = refinement.AbstractRefinement(cfg["pipeline"][input_step]) # type: ignore[abstract] + refinement_ = refinement.AbstractRefinement( + cfg["pipeline"][input_step], self.step, self.window_size + ) # type: ignore[abstract] self.pipeline_cfg["pipeline"][input_step] = refinement_.cfg self.margins.add_non_cumulative(input_step, refinement_.margins) @@ -299,6 +338,34 @@ def matching_cost_prepare(self, cfg: Dict[str, dict], input_step: str) -> None: self.left_img, self.right_img, self.disp_min_col, self.disp_max_col, cfg["pipeline"][input_step] ) + def estimation_run(self, cfg: Dict[str, dict], input_step: str) -> None: + """ + Shift's estimation step + + :param cfg: pipeline configuration + :type cfg: dict + :param input_step: step to trigger + :type input_step: str + :return: None + """ + + logging.info("Estimation computation...") + estimation_ = estimation.AbstractEstimation(cfg["pipeline"][input_step]) # type: ignore[abstract] + + row_disparity, col_disparity, shifts, extra_dict = estimation_.compute_estimation(self.left_img, self.right_img) + + self.left_img = img_tools.add_disparity_grid(self.left_img, col_disparity, row_disparity) + # Column's min, max disparities + self.disp_min_col = self.left_img["col_disparity"].sel(band_disp="min").data + self.disp_max_col = self.left_img["col_disparity"].sel(band_disp="max").data + # Row's min, max disparities + self.disp_min_row = self.left_img["row_disparity"].sel(band_disp="min").data + self.disp_max_row = self.left_img["row_disparity"].sel(band_disp="max").data + + self.completed_cfg = estimation_.update_cfg_with_estimation( + cfg, col_disparity, row_disparity, shifts, extra_dict + ) + def matching_cost_run(self, _, __) -> None: """ Matching cost computation @@ -332,7 +399,9 @@ def disp_maps_run(self, cfg: Dict[str, dict], input_step: str) -> None: disparity_run = disparity.Disparity(cfg["pipeline"][input_step]) map_col, map_row = disparity_run.compute_disp_maps(self.cost_volumes) - self.dataset_disp_maps = common.dataset_disp_maps(map_row, map_col) + self.dataset_disp_maps = common.dataset_disp_maps( + map_row, map_col, {"invalid_disp": cfg["pipeline"]["disparity"]["invalid_disparity"]} + ) def refinement_run(self, cfg: Dict[str, dict], input_step: str) -> None: """ @@ -346,7 +415,15 @@ def refinement_run(self, cfg: Dict[str, dict], input_step: str) -> None: """ logging.info("Refinement computation...") - refinement_run = refinement.AbstractRefinement(cfg["pipeline"][input_step]) # type: ignore[abstract] - refine_map_col, refine_map_row = refinement_run.refinement_method(self.cost_volumes, self.dataset_disp_maps) + if cfg["pipeline"][input_step]["refinement_method"] == "optical_flow": + logging.warning("The optical flow method is still in an experimental phase.") + + refinement_run = refinement.AbstractRefinement( + cfg["pipeline"][input_step], self.step, self.window_size + ) # type: ignore[abstract] + + refine_map_col, refine_map_row = refinement_run.refinement_method( + self.cost_volumes, self.dataset_disp_maps, self.left_img, self.right_img + ) self.dataset_disp_maps = common.dataset_disp_maps(refine_map_row, refine_map_col) diff --git a/setup.cfg b/setup.cfg index 055f405..a5ee30d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,5 @@ # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of Pandora2D # (see https://github.com/CNES/Pandora2d). @@ -64,6 +65,7 @@ install_requires = numba>=0.47.0;python_version<'3.8' pandora==1.6.* typing_extensions + scikit-image package_dir = . = pandora2d diff --git a/tests/common.py b/tests/common.py index 965b28e..f42a3d5 100644 --- a/tests/common.py +++ b/tests/common.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -68,6 +69,7 @@ correct_pipeline = { "pipeline": { + "estimation": {"estimation_method": "phase_cross_correlation"}, "matching_cost": {"matching_cost_method": "zncc", "window_size": 5}, "disparity": {"disparity_method": "wta", "invalid_disparity": -99}, "refinement": {"refinement_method": "interpolation"}, diff --git a/tests/test_common.py b/tests/test_common.py index 56cea77..17b7783 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -23,75 +24,65 @@ """ Test common """ + +# pylint: disable=redefined-outer-name import os -import unittest -import xarray as xr import numpy as np +import pytest +import xarray as xr from pandora2d import common -class TestCommon(unittest.TestCase): +@pytest.fixture +def create_test_dataset(): + """ + Create a test dataset + """ + row, col = np.ones((2, 2)), np.ones((2, 2)) + + dataset_y = xr.Dataset( + {"row_map": (["row", "col"], row)}, + coords={"row": np.arange(row.shape[0]), "col": np.arange(row.shape[1])}, + ) + + dataset_x = xr.Dataset( + {"col_map": (["row", "col"], col)}, + coords={"row": np.arange(col.shape[0]), "col": np.arange(col.shape[1])}, + ) + + dataset = dataset_y.merge(dataset_x, join="override", compat="override") + + return dataset + + +def test_save_dataset(create_test_dataset): """ - TestImgTools class allows to test all the methods in the img_tools function + Function for testing the dataset_save function """ - def setUp(self) -> None: - """ - Method called to prepare the test fixture - - """ - self.row = np.ones((2, 2)) - self.col = np.ones((2, 2)) - - def test_save_dataset(self): - """ - Function for testing the dataset_save function - """ - dataset_y = xr.Dataset( - {"row_map": (["row", "col"], self.row)}, - coords={"row": np.arange(self.row.shape[0]), "col": np.arange(self.row.shape[1])}, - ) - - dataset_x = xr.Dataset( - {"col_map": (["row", "col"], self.col)}, - coords={"row": np.arange(self.col.shape[0]), "col": np.arange(self.col.shape[1])}, - ) - - dataset = dataset_y.merge(dataset_x, join="override", compat="override") - - common.save_dataset(dataset, "./tests/res_test/") - assert os.path.exists("./tests/res_test/") - - assert os.path.exists("./tests/res_test/columns_disparity.tif") - assert os.path.exists("./tests/res_test/row_disparity.tif") - - os.remove("./tests/res_test/columns_disparity.tif") - os.remove("./tests/res_test/row_disparity.tif") - os.rmdir("./tests/res_test") - - @staticmethod - def test_dataset_disp_maps(): - """ - Test function for create a dataset - """ - - # create a test dataset for map row - data = np.zeros((2, 2)) - dataset_y = xr.Dataset( - {"row_map": (["row", "col"], data)}, - coords={"row": np.arange(data.shape[0]), "col": np.arange(data.shape[1])}, - ) - # create a test dataset for map col - dataset_x = xr.Dataset( - {"col_map": (["row", "col"], data)}, - coords={"row": np.arange(data.shape[0]), "col": np.arange(data.shape[1])}, - ) - # merge two dataset into one - dataset_test = dataset_y.merge(dataset_x, join="override", compat="override") - - # create dataset with function - dataset_fun = common.dataset_disp_maps(data, data) - - assert dataset_fun.equals(dataset_test) + common.save_dataset(create_test_dataset, "./tests/res_test/") + assert os.path.exists("./tests/res_test/") + + assert os.path.exists("./tests/res_test/columns_disparity.tif") + assert os.path.exists("./tests/res_test/row_disparity.tif") + + os.remove("./tests/res_test/columns_disparity.tif") + os.remove("./tests/res_test/row_disparity.tif") + os.rmdir("./tests/res_test") + + +def test_dataset_disp_maps(create_test_dataset): + """ + Test function for create a dataset + """ + + dataset_test = create_test_dataset + + dataset_test.attrs = {"invalid_disp": -9999} + + # create dataset with function + dataset_fun = common.dataset_disp_maps(np.ones((2, 2)), np.ones((2, 2)), {"invalid_disp": -9999}) + + assert dataset_fun.equals(dataset_test) diff --git a/tests/test_dichotomy.py b/tests/test_dichotomy.py index ba68da0..70ca24d 100644 --- a/tests/test_dichotomy.py +++ b/tests/test_dichotomy.py @@ -120,7 +120,7 @@ def test_refinement_method(config, caplog, mocker: MockerFixture): dichotomy_instance = refinement.dichotomy.Dichotomy(config) # We can pass anything as it is not yet implemented - dichotomy_instance.refinement_method(mocker.ANY, mocker.ANY) + dichotomy_instance.refinement_method(mocker.ANY, mocker.ANY, mocker.ANY, mocker.ANY) assert "refinement_method of Dichotomy not yet implemented" in caplog.messages diff --git a/tests/test_estimation.py b/tests/test_estimation.py new file mode 100644 index 0000000..aac15f7 --- /dev/null +++ b/tests/test_estimation.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python +# +# Copyright (c) 2024 CS GROUP France +# +# This file is part of PANDORA2D +# +# https://github.com/CNES/Pandora2D +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Test Matching cost class +""" + +# pylint: disable=redefined-outer-name + + +import numpy as np +import pytest +import json_checker +import xarray as xr +from pandora2d import estimation + + +@pytest.fixture() +def estimation_class(): + """Build estimation object.""" + estimation_class = estimation.AbstractEstimation( # type: ignore[abstract] + { + "estimation_method": "phase_cross_correlation", + "range_col": 5, + "range_row": 5, + "sample_factor": 10, + } + ) + return estimation_class + + +@pytest.fixture() +def stereo_object(): + """Create a stereo object""" + + # Create a stereo object + data = np.array( + ([-9999, -9999, -9999], [1, 1, 1], [3, 4, 5]), + dtype=np.float64, + ) + mask = np.array(([1, 1, 1], [0, 0, 0], [0, 0, 0]), dtype=np.int16) + left = xr.Dataset( + {"im": (["row", "col"], data), "msk": (["row", "col"], mask)}, + coords={"row": np.arange(data.shape[0]), "col": np.arange(data.shape[1])}, + ) + left.attrs = { + "no_data_img": -9999, + "valid_pixels": 0, + "no_data_mask": 1, + "crs": None, + "col_disparity_source": [-1, 0], + "row_disparity_source": [-1, 0], + } + + data = np.array( + ([1, 1, 1], [3, 4, 5], [1, 1, 1]), + dtype=np.float64, + ) + mask = np.array(([0, 0, 0], [0, 0, 0], [0, 0, 0]), dtype=np.int16) + right = xr.Dataset( + {"im": (["row", "col"], data), "msk": (["row", "col"], mask)}, + coords={"row": np.arange(data.shape[0]), "col": np.arange(data.shape[1])}, + ) + right.attrs = { + "no_data_img": -9999, + "valid_pixels": 0, + "no_data_mask": 1, + "crs": None, + } + + return left, right + + +@pytest.mark.parametrize( + ["estimation_method", "range_col", "range_row", "sample_factor", "error"], + [ + pytest.param("another_method", 5, 5, 10, KeyError, id="Wrong method"), + pytest.param( + "phase_cross_correlation", -1, 5, 10, json_checker.core.exceptions.DictCheckerError, id="negative range_col" + ), + pytest.param( + "phase_cross_correlation", "5", 5, 10, json_checker.core.exceptions.DictCheckerError, id="string range_col" + ), + pytest.param( + "phase_cross_correlation", 5, 0, 10, json_checker.core.exceptions.DictCheckerError, id="0 as range_row" + ), + pytest.param( + "phase_cross_correlation", 5, [2, 3], 10, json_checker.core.exceptions.DictCheckerError, id="list range_row" + ), + pytest.param( + "phase_cross_correlation", 5, 5, 0, json_checker.core.exceptions.DictCheckerError, id="0 for sample factor" + ), + pytest.param( + "phase_cross_correlation", + 5, + 5, + 15, + json_checker.core.exceptions.DictCheckerError, + id="not a multiple of 10 sample factor", + ), + ], +) +def test_false_check_conf(estimation_method, range_col, range_row, sample_factor, error): + """ + test check_conf of estimation with wrongs pipelines + """ + + with pytest.raises(error): + estimation.AbstractEstimation( + { + "estimation_method": estimation_method, + "range_col": range_col, + "range_row": range_row, + "sample_factor": sample_factor, + } + ) # type: ignore[abstract] + + +def test_check_conf(): + """ + test check_conf of estimation with a correct pipeline + """ + estimation.AbstractEstimation( + { + "estimation_method": "phase_cross_correlation", + "range_col": 5, + "range_row": 5, + "sample_factor": 10, + } + ) # type: ignore[abstract] + + +def test_update_cfg_with_estimation(estimation_class): + """ + test update_cfg_with_estimation function + """ + + gt_cfg = { + "input": {"col_disparity": [-2, 2], "row_disparity": [-2, 2]}, + "pipeline": {"estimation": {"estimated_shifts": [-0.5, 1.3], "error": [1.0], "phase_diff": [1.0]}}, + } + + cfg = estimation_class.update_cfg_with_estimation( + {"input": {}, "pipeline": {"estimation": {}}}, + [-2, 2], + [-2, 2], + -np.array([0.5, -1.3]), + {"error": np.array([1.0]), "phase_diff": np.array([1.0])}, + ) + + assert gt_cfg == cfg + + +def test_estimation_computation(stereo_object, estimation_class): + """ + test estimation_computation with phase_cross_correlation + """ + + left, right = stereo_object + estimation_ = estimation_class + + row_disparity, col_disparity, shifts, extra_dict = estimation_.compute_estimation(left, right) + + assert col_disparity == [-5, 5] + assert row_disparity == [-6, 4] + assert np.array_equal(shifts, [-0.8, 0]) + assert extra_dict["error"] == 0.9999999999855407 + assert extra_dict["phase_diff"] == "1.06382330e-18" diff --git a/tests/test_img_tools/test_create_datasets_from_input.py b/tests/test_img_tools/test_create_datasets_from_input.py index 5055093..00c92f8 100644 --- a/tests/test_img_tools/test_create_datasets_from_input.py +++ b/tests/test_img_tools/test_create_datasets_from_input.py @@ -1,4 +1,5 @@ # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -165,3 +166,21 @@ def test_fails_when_disparity_max_lt_disparity_min(self, input_section, disparit with pytest.raises(ValueError) as exc_info: img_tools.create_datasets_from_inputs(input_section) assert exc_info.value.args[0] == "Min disparity (8) should be lower than Max disparity (-10)" + + def test_create_dataset_from_inputs_with_estimation_step(self, input_section): + """ + test dataset_from_inputs with an estimation step and no disparity range + """ + + configuration_with_estimation = {"input": input_section} + del configuration_with_estimation["input"]["row_disparity"] + del configuration_with_estimation["input"]["col_disparity"] + configuration_with_estimation["pipeline"] = {"estimation": {"estimation_method": "phase_cross_correlation"}} + result = img_tools.create_datasets_from_inputs( + input_section, estimation_cfg=configuration_with_estimation["pipeline"].get("estimation") + ) + + assert result.left.attrs["col_disparity_source"] == [-9999, -9999] + assert result.left.attrs["row_disparity_source"] == [-9999, -9999] + assert result.right.attrs["col_disparity_source"] == [9999, 9999] + assert result.right.attrs["row_disparity_source"] == [9999, 9999] diff --git a/tests/test_matching_cost.py b/tests/test_matching_cost.py index 70034b6..ae19089 100644 --- a/tests/test_matching_cost.py +++ b/tests/test_matching_cost.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -387,6 +388,7 @@ def test_allocate_cost_volume(self): cost_volumes_test.attrs["no_data_img"] = -9999 cost_volumes_test.attrs["no_data_mask"] = 1 cost_volumes_test.attrs["valid_pixels"] = 0 + cost_volumes_test.attrs["step"] = [1, 1] # data by function compute_cost_volume cfg = {"matching_cost_method": "zncc", "window_size": 3} diff --git a/tests/test_pandora2d.py b/tests/test_pandora2d.py index d2cdb82..e6fc596 100644 --- a/tests/test_pandora2d.py +++ b/tests/test_pandora2d.py @@ -2,6 +2,7 @@ # coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -74,10 +75,11 @@ def test_run_prepare() -> None: } img_left, img_right = create_datasets_from_inputs(input_config=input_config) - pandora2d_machine.run_prepare(img_left, img_right) + pandora2d_machine.run_prepare(img_left, img_right, input_config) assert pandora2d_machine.left_img == img_left assert pandora2d_machine.right_img == img_right + assert pandora2d_machine.completed_cfg == input_config np.testing.assert_array_equal( pandora2d_machine.disp_min_col, np.full((img_left.sizes["row"], img_left.sizes["col"]), -2) ) diff --git a/tests/test_refinement.py b/tests/test_refinement.py index e1a886f..8d6fc68 100644 --- a/tests/test_refinement.py +++ b/tests/test_refinement.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# coding: utf8 # # Copyright (c) 2021 Centre National d'Etudes Spatiales (CNES). +# Copyright (c) 2024 CS GROUP France # # This file is part of PANDORA2D # @@ -24,126 +24,277 @@ Test refinement step """ -import unittest +# pylint: disable=redefined-outer-name, protected-access +# mypy: disable-error-code=attr-defined + + import numpy as np import xarray as xr import pytest +from json_checker.core.exceptions import DictCheckerError + from pandora.margins import Margins from pandora2d import refinement, common -class TestRefinement(unittest.TestCase): +@pytest.fixture() +def cv_dataset(): + """ + Create dataset cost volumes + """ + + cv = np.zeros((3, 3, 5, 5)) + cv[:, :, 2, 2] = np.ones([3, 3]) + cv[:, :, 2, 3] = np.ones([3, 3]) + cv[:, :, 3, 2] = np.ones([3, 3]) + cv[:, :, 3, 3] = np.ones([3, 3]) + + c_row = np.arange(cv.shape[0]) + c_col = np.arange(cv.shape[1]) + + # First pixel in the image that is fully computable (aggregation windows are complete) + row = np.arange(c_row[0], c_row[-1] + 1) + col = np.arange(c_col[0], c_col[-1] + 1) + + disparity_range_col = np.arange(-2, 2 + 1) + disparity_range_row = np.arange(-2, 2 + 1) + + cost_volumes_test = xr.Dataset( + {"cost_volumes": (["row", "col", "disp_col", "disp_row"], cv)}, + coords={"row": row, "col": col, "disp_col": disparity_range_col, "disp_row": disparity_range_row}, + ) + + cost_volumes_test.attrs["measure"] = "zncc" + cost_volumes_test.attrs["window_size"] = 1 + cost_volumes_test.attrs["type_measure"] = "max" + + return cost_volumes_test + + +@pytest.fixture() +def dataset_image(): + """ + Create an image dataset + """ + data = np.arange(30).reshape((6, 5)) + + img = xr.Dataset( + {"im": (["row", "col"], data)}, + coords={"row": np.arange(data.shape[0]), "col": np.arange(data.shape[1])}, + ) + img.attrs = { + "no_data_img": -9999, + "valid_pixels": 0, + "no_data_mask": 1, + "crs": None, + "col_disparity_source": [-2, 2], + "row_disparity_source": [-2, 2], + "invalid_disparity": np.nan, + } + + return img + + +def test_margins(): + """ + test margins of matching cost pipeline + """ + _refinement = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] + + assert _refinement.margins == Margins(3, 3, 3, 3), "Not a cubic kernel Margins" + + +@pytest.mark.parametrize("refinement_method", ["interpolation", "optical_flow"]) +def test_check_conf_passes(refinement_method): + """ + Test the check_conf function + """ + refinement.AbstractRefinement({"refinement_method": refinement_method}) # type: ignore[abstract] + + +@pytest.mark.parametrize( + "refinement_config", [{"refinement_method": "wta"}, {"refinement_method": "optical_flow", "iterations": 0}] +) +def test_check_conf_fails(refinement_config): + """ + Test the refinement check_conf with wrong configuration + """ + + with pytest.raises((KeyError, DictCheckerError)): + refinement.AbstractRefinement(refinement_config) # type: ignore[abstract] + + +def test_refinement_method_subpixel(cv_dataset): + """ + test refinement_method with interpolation + """ + + cost_volumes_test = cv_dataset + + data = np.full((3, 3), 0.4833878) + + dataset_disp_map = common.dataset_disp_maps(data, data) + + test = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] + delta_x, delta_y = test.refinement_method(cost_volumes_test, dataset_disp_map, None, None) + + np.testing.assert_allclose(data, delta_y, rtol=1e-06) + np.testing.assert_allclose(data, delta_x, rtol=1e-06) + + +def test_refinement_method_pixel(cv_dataset): """ - TestRefinement class allows to test the refinement module + test refinement """ - @staticmethod - def test_check_conf(): - """ - Test the interpolation method - """ + cost_volumes_test = cv_dataset - refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] + new_cv_datas = np.zeros((3, 3, 5, 5)) + new_cv_datas[:, :, 1, 3] = np.ones([3, 3]) - with pytest.raises(KeyError): - refinement.AbstractRefinement({"refinement_method": "wta"}) # type: ignore[abstract] + cost_volumes_test["cost_volumes"].data = new_cv_datas - @staticmethod - def test_margins(): - """ - test margins of matching cost pipeline - """ - _refinement = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] + gt_delta_y = np.array( + ([[-1, -1, -1], [-1, -1, -1], [-1, -1, -1]]), + dtype=np.float64, + ) - assert _refinement.margins == Margins(3, 3, 3, 3), "Not a cubic kernel Margins" + gt_delta_x = np.array( + ([[1, 1, 1], [1, 1, 1], [1, 1, 1]]), + dtype=np.float64, + ) - @staticmethod - def test_refinement_method_subpixel(): - """ - test refinement - """ + dataset_disp_map = common.dataset_disp_maps(gt_delta_y, gt_delta_x) - cv = np.zeros((3, 3, 5, 5)) - cv[:, :, 2, 2] = np.ones([3, 3]) - cv[:, :, 2, 3] = np.ones([3, 3]) - cv[:, :, 3, 2] = np.ones([3, 3]) - cv[:, :, 3, 3] = np.ones([3, 3]) + test = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] + delta_x, delta_y = test.refinement_method(cost_volumes_test, dataset_disp_map, None, None) - c_row = [0, 1, 2] - c_col = [0, 1, 2] + np.testing.assert_allclose(gt_delta_y, delta_y, rtol=1e-06) + np.testing.assert_allclose(gt_delta_x, delta_x, rtol=1e-06) - # First pixel in the image that is fully computable (aggregation windows are complete) - row = np.arange(c_row[0], c_row[-1] + 1) - col = np.arange(c_col[0], c_col[-1] + 1) - disparity_range_col = np.arange(-2, 2 + 1) - disparity_range_row = np.arange(-2, 2 + 1) +def test_optical_flow_margins(): + """ + test get_margins of refinement pipeline + """ + gt = (2, 2, 2, 2) # with 5 has default window size + _refinement = refinement.AbstractRefinement({"refinement_method": "optical_flow"}) # type: ignore[abstract] + + r_margins = _refinement.margins.astuple() + + assert r_margins == gt + + +def test_reshape_to_matching_cost_window_left(dataset_image): + """ + Test reshape_to_matching_cost_window function for a left image + """ - cost_volumes_test = xr.Dataset( - {"cost_volumes": (["row", "col", "disp_col", "disp_row"], cv)}, - coords={"row": row, "col": col, "disp_col": disparity_range_col, "disp_row": disparity_range_row}, - ) + img = dataset_image - cost_volumes_test.attrs["measure"] = "zncc" - cost_volumes_test.attrs["window_size"] = 1 - cost_volumes_test.attrs["type_measure"] = "max" + refinement_class = refinement.AbstractRefinement({"refinement_method": "optical_flow"}) # type: ignore[abstract] + refinement_class._window_size = 3 - data = np.array( - ([[0.4833878, 0.4833878, 0.4833878], [0.4833878, 0.4833878, 0.4833878], [0.4833878, 0.4833878, 0.4833878]]), - dtype=np.float64, - ) + cv = np.zeros((6, 5, 5, 5)) - dataset_disp_map = common.dataset_disp_maps(data, data) + disparity_range_col = np.arange(-2, 2 + 1) + disparity_range_row = np.arange(-2, 2 + 1) - test = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] - delta_x, delta_y = test.refinement_method(cost_volumes_test, dataset_disp_map) + cost_volumes = xr.Dataset( + {"cost_volumes": (["row", "col", "disp_col", "disp_row"], cv)}, + coords={ + "row": np.arange(0, 6), + "col": np.arange(0, 5), + "disp_col": disparity_range_col, + "disp_row": disparity_range_row, + }, + ) - np.testing.assert_allclose(data, delta_y, rtol=1e-06) - np.testing.assert_allclose(data, delta_x, rtol=1e-06) + # for left image + reshaped_left = refinement_class.reshape_to_matching_cost_window(img, cost_volumes) - @staticmethod - def test_refinement_method_pixel(): - """ - test refinement - """ + # test four matching_cost + idx_1_1 = [[0, 1, 2], [5, 6, 7], [10, 11, 12]] + idx_2_2 = [[6, 7, 8], [11, 12, 13], [16, 17, 18]] + idx_3_3 = [[12, 13, 14], [17, 18, 19], [22, 23, 24]] + idx_4_1 = [[15, 16, 17], [20, 21, 22], [25, 26, 27]] - cv = np.zeros((3, 3, 5, 5)) - cv[:, :, 1, 3] = np.ones([3, 3]) + assert np.array_equal(reshaped_left[:, :, 0], idx_1_1) + assert np.array_equal(reshaped_left[:, :, 4], idx_2_2) + assert np.array_equal(reshaped_left[:, :, 8], idx_3_3) + assert np.array_equal(reshaped_left[:, :, 9], idx_4_1) - c_row = [0, 1, 2] - c_col = [0, 1, 2] - # First pixel in the image that is fully computable (aggregation windows are complete) - row = np.arange(c_row[0], c_row[-1] + 1) - col = np.arange(c_col[0], c_col[-1] + 1) +def test_reshape_to_matching_cost_window_right(dataset_image): + """ + Test reshape_to_matching_cost_window function for a right image + """ + + img = dataset_image + + refinement_class = refinement.AbstractRefinement({"refinement_method": "optical_flow"}) # type: ignore[abstract] + refinement_class._window_size = 3 + + # Create disparity maps + col_disp_map = [2, 0, 0, 0, 1, 0, 0, 0, 1, -2, 0, 0] + row_disp_map = [2, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0] + + cv = np.zeros((6, 5, 5, 5)) + + disparity_range_col = np.arange(-2, 2 + 1) + disparity_range_row = np.arange(-2, 2 + 1) + + cost_volumes = xr.Dataset( + {"cost_volumes": (["row", "col", "disp_col", "disp_row"], cv)}, + coords={ + "row": np.arange(0, 6), + "col": np.arange(0, 5), + "disp_col": disparity_range_col, + "disp_row": disparity_range_row, + }, + ) + + # for right image + reshaped_right = refinement_class.reshape_to_matching_cost_window(img, cost_volumes, row_disp_map, col_disp_map) + + # test four matching_cost + idx_1_1 = [[12, 13, 14], [17, 18, 19], [22, 23, 24]] + idx_2_2 = [[2, 3, 4], [7, 8, 9], [12, 13, 14]] + + assert np.array_equal(reshaped_right[:, :, 0], idx_1_1) + assert np.array_equal(reshaped_right[:, :, 4], idx_2_2) + + +def test_warped_image_without_step(): + """ + test warped image + """ - disparity_range_col = np.arange(-2, 2 + 1) - disparity_range_row = np.arange(-2, 2 + 1) + refinement_class = refinement.AbstractRefinement({"refinement_method": "optical_flow"}) # type: ignore[abstract] - cost_volumes_test = xr.Dataset( - {"cost_volumes": (["row", "col", "disp_col", "disp_row"], cv)}, - coords={"row": row, "col": col, "disp_col": disparity_range_col, "disp_row": disparity_range_row}, - ) + mc_1 = np.array( + [[0, 1, 2, 3, 4], [6, 7, 8, 9, 10], [12, 13, 14, 15, 16], [18, 19, 20, 21, 22], [24, 25, 26, 27, 28]] + ) + mc_2 = np.array( + [[1, 2, 3, 4, 5], [7, 8, 9, 10, 11], [13, 14, 15, 16, 17], [19, 20, 21, 22, 23], [25, 26, 27, 28, 29]] + ) - cost_volumes_test.attrs["measure"] = "zncc" - cost_volumes_test.attrs["window_size"] = 1 - cost_volumes_test.attrs["type_measure"] = "max" + reshaped_right = np.stack((mc_1, mc_2)).transpose((1, 2, 0)) - gt_delta_y = np.array( - ([[-1, -1, -1], [-1, -1, -1], [-1, -1, -1]]), - dtype=np.float64, - ) + delta_row = -3 * np.ones(2) + delta_col = -np.ones(2) - gt_delta_x = np.array( - ([[1, 1, 1], [1, 1, 1], [1, 1, 1]]), - dtype=np.float64, - ) + test_img_shift = refinement_class.warped_img(reshaped_right, delta_row, delta_col, [0, 1]) - dataset_disp_map = common.dataset_disp_maps(gt_delta_y, gt_delta_x) + gt_mc_1 = np.array( + [[19, 20, 21, 22, 22], [25, 26, 27, 28, 28], [25, 26, 27, 28, 28], [19, 20, 21, 22, 22], [13, 14, 15, 16, 16]] + ) - test = refinement.AbstractRefinement({"refinement_method": "interpolation"}) # type: ignore[abstract] - delta_x, delta_y = test.refinement_method(cost_volumes_test, dataset_disp_map) + gt_mc_2 = np.array( + [[20, 21, 22, 23, 23], [26, 27, 28, 29, 29], [26, 27, 28, 29, 29], [20, 21, 22, 23, 23], [14, 15, 16, 17, 17]] + ) - np.testing.assert_allclose(gt_delta_y, delta_y, rtol=1e-06) - np.testing.assert_allclose(gt_delta_x, delta_x, rtol=1e-06) + # check that the generated image is equal to ground truth + assert np.array_equal(gt_mc_1, test_img_shift[:, :, 0]) + assert np.array_equal(gt_mc_2, test_img_shift[:, :, 1])