Merge branch '11-pandora_version_1_5_0' into 'master'

fix: remove get_config_pipeline Closes #11 See merge request 3d/PandoraBox/pandora2d!29
CNES · Apr 28, 2023 · 1f934e5 · 1f934e5
2 parents 942f887 + 2e3ce17
commit 1f934e5
Show file tree

Hide file tree

Showing 8 changed files with 76 additions and 58 deletions.
diff --git a/data_samples/json_conf_files/a_basic_pipeline.json b/data_samples/json_conf_files/a_basic_pipeline.json
@@ -5,10 +5,10 @@
     "img_right": "./maricopa/right.tif",
     "nodata_right": -9999,
 
-    "disp_min_x": -2,
-    "disp_max_x": 2,
-    "disp_min_y": -2,
-    "disp_max_y": 2
+    "disp_min_col": -2,
+    "disp_max_col": 2,
+    "disp_min_row": -2,
+    "disp_max_row": 2
   },
   "pipeline": {
     "matching_cost": {

diff --git a/notebooks/introduction_and_basic_usage.ipynb b/notebooks/introduction_and_basic_usage.ipynb
@@ -198,17 +198,6 @@
     "img_right_path = \"data/right.tif\""
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "essential-welsh",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# image configuration\n",
-    "image_cfg = {'image': {'no_data_left': np.nan, 'no_data_right': np.nan}}"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "integral-arizona",
@@ -244,8 +233,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "img_left = read_img(img_left_path, no_data=image_cfg['image']['no_data_left'])\n",
-    "img_right = read_img(img_right_path, no_data=image_cfg['image']['no_data_right'])"
+    "img_left = read_img(img_left_path, no_data=np.nan)\n",
+    "img_right = read_img(img_right_path, no_data=np.nan)"
    ]
   },
   {
@@ -315,8 +304,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "user_pipeline_cfg = {\n",
-    "    'pipeline':{\n",
+    "user_cfg = {\n",
+    "    \"input\": {\n",
+    "        \"img_left\": \"data/left.png\",\n",
+    "        \"img_right\": \"data/right.png\",\n",
+    "        \"nodata_left\": \"NaN\",\n",
+    "        \"disp_min_col\": -2,\n",
+    "        \"disp_max_col\": 2,\n",
+    "        \"disp_min_row\": -2,\n",
+    "        \"disp_max_row\": 2,\n",
+    "    },\n",
+    "    \"pipeline\":{\n",
     "        \"matching_cost\" : {\n",
     "            \"matching_cost_method\": \"zncc\",\n",
     "            \"window_size\": 5,\n",
@@ -385,7 +383,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "checked_cfg = check_json.check_pipeline_section(user_pipeline_cfg, pandora2d_machine)"
+    "checked_cfg = check_json.check_pipeline_section(user_cfg, pandora2d_machine)"
    ]
   },
   {
@@ -432,7 +430,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dataset = pandora2d.run(pandora2d_machine, img_left, img_right, disp_min_col, disp_max_col, disp_min_row, disp_max_row, pipeline_cfg)"
+    "dataset = pandora2d.run(pandora2d_machine, img_left, img_right, disp_min_col, disp_max_col, disp_min_row, disp_max_row, user_cfg)"
    ]
   },
   {
@@ -458,7 +456,7 @@
    "execution_count": null,
    "id": "vocational-africa",
    "metadata": {
-    "scrolled": true
+    "scrolled": false
    },
    "outputs": [],
    "source": [
@@ -537,8 +535,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "user_pipeline_cfg = {\n",
-    "    'pipeline':{\n",
+    "user_cfg = {\n",
+    "    \"input\": {\n",
+    "        \"img_left\": \"data/left.png\",\n",
+    "        \"img_right\": \"data/right.png\",\n",
+    "        \"nodata_left\": \"NaN\",\n",
+    "        \"disp_min_col\": -2,\n",
+    "        \"disp_max_col\": 2,\n",
+    "        \"disp_min_row\": -2,\n",
+    "        \"disp_max_row\": 2,\n",
+    "    },\n",
+    "    \"pipeline\":{\n",
     "        \"matching_cost\" : {\n",
     "            \"matching_cost_method\": \"zncc\",\n",
     "            \"window_size\": 5,\n",
@@ -607,7 +614,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "checked_cfg = check_json.check_pipeline_section(user_pipeline_cfg, pandora2d_machine)"
+    "checked_cfg = check_json.check_pipeline_section(user_cfg, pandora2d_machine)"
    ]
   },
   {
@@ -676,7 +683,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pandora2d_machine.run('matching_cost', pipeline_cfg)"
+    "pandora2d_machine.run('matching_cost', user_cfg)"
    ]
   },
   {
@@ -704,7 +711,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pandora2d_machine.run('disparity', pipeline_cfg)"
+    "pandora2d_machine.run('disparity', user_cfg)"
    ]
   },
   {
@@ -754,7 +761,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pandora2d_machine.run('refinement', pipeline_cfg)"
+    "pandora2d_machine.run('refinement', user_cfg)"
    ]
   },
   {
@@ -806,9 +813,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "env_pandora_2D",
+   "display_name": "pandora-venv",
    "language": "python",
-   "name": "env_pandora_2d"
+   "name": "pandora-venv"
   },
   "language_info": {
    "codemirror_mode": {
@@ -820,7 +827,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.9"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,

diff --git a/pandora2d/__init__.py b/pandora2d/__init__.py
@@ -74,7 +74,7 @@ def run(
 
     pandora2d_machine.run_prepare(img_left, img_right, disp_min_col, disp_max_col, disp_min_row, disp_max_row)
 
-    for e in list(cfg_pipeline):
+    for e in list(cfg_pipeline["pipeline"]):
         pandora2d_machine.run(e, cfg_pipeline)
 
     pandora2d_machine.run_exit()
@@ -107,18 +107,18 @@ def main(cfg_path: str, path_output: str, verbose: bool) -> None:
     img_left = read_img(cfg["input"]["img_left"], cfg["input"]["nodata_left"])
     img_right = read_img(cfg["input"]["img_right"], cfg["input"]["nodata_right"])
 
-    ## read disparities values
+    # read disparities values
     disp_min_col = cfg["input"]["disp_min_col"]
     disp_max_col = cfg["input"]["disp_max_col"]
     disp_min_row = cfg["input"]["disp_min_row"]
     disp_max_row = cfg["input"]["disp_max_row"]
 
     # run pandora 2D and store disp maps in a dataset
     dataset_disp_maps = run(
-        pandora2d_machine, img_left, img_right, disp_min_col, disp_max_col, disp_min_row, disp_max_row, cfg["pipeline"]
+        pandora2d_machine, img_left, img_right, disp_min_col, disp_max_col, disp_min_row, disp_max_row, cfg
     )
 
     # save dataset
     common.save_dataset(dataset_disp_maps, path_output)
-    #save config
+    # save config
     save_config(path_output, user_cfg)
diff --git a/pandora2d/check_json.py b/pandora2d/check_json.py
@@ -29,7 +29,7 @@
 import numpy as np
 
 from pandora.check_json import check_disparities, check_images, get_config_input
-from pandora.check_json import get_config_pipeline, concat_conf, update_conf, rasterio_can_open_mandatory
+from pandora.check_json import concat_conf, update_conf, rasterio_can_open_mandatory
 
 
 from pandora2d.state_machine import Pandora2DMachine
@@ -78,16 +78,20 @@ def check_pipeline_section(user_cfg: Dict[str, dict], pandora2d_machine: Pandora
     """
 
     cfg = update_conf({}, user_cfg)
-    pandora2d_machine.check_conf(cfg["pipeline"])
+    pandora2d_machine.check_conf(cfg)
 
     cfg = update_conf(cfg, pandora2d_machine.pipeline_cfg)
 
     configuration_schema = {"pipeline": dict}
 
     checker = Checker(configuration_schema)
-    checker.validate(cfg)
 
-    return cfg
+    # We select only the pipeline section for the checker
+    pipeline_cfg = {"pipeline": cfg["pipeline"]}
+
+    checker.validate(pipeline_cfg)
+
+    return pipeline_cfg
 
 
 def check_conf(user_cfg: Dict[str, dict], pandora2d_machine: Pandora2DMachine) -> dict:
@@ -107,8 +111,7 @@ def check_conf(user_cfg: Dict[str, dict], pandora2d_machine: Pandora2DMachine) -
     cfg_input = check_input_section(user_cfg_input)
 
     # check pipeline
-    user_cfg_pipeline = get_config_pipeline(user_cfg)
-    cfg_pipeline = check_pipeline_section(user_cfg_pipeline, pandora2d_machine)
+    cfg_pipeline = check_pipeline_section(user_cfg, pandora2d_machine)
 
     if isinstance(cfg_input["input"]["nodata_right"], float) and cfg_pipeline["pipeline"]["matching_cost"][
         "matching_cost_method"

diff --git a/pandora2d/disparity/disparity.py b/pandora2d/disparity/disparity.py
@@ -51,6 +51,8 @@ def check_conf(self, **cfg: str) -> Dict[str, str]:
         # Give the default value if the required element is not in the configuration
         if "invalid_disparity" not in cfg:
             cfg["invalid_disparity"] = self._INVALID_DISPARITY  # type: ignore
+        elif cfg["invalid_disparity"] == "NaN":
+            cfg["invalid_disparity"] = np.nan
 
         schema = {
             "disparity_method": And(str, lambda x: x in ["wta"]),

diff --git a/pandora2d/state_machine.py b/pandora2d/state_machine.py
@@ -196,7 +196,7 @@ def check_conf(self, cfg: Dict[str, dict]) -> None:
         # Add transitions to the empty machine.
         self.add_transitions(self._transitions_check)
 
-        for input_step in list(cfg):
+        for input_step in list(cfg["pipeline"]):
             try:
                 self.trigger(input_step, cfg, input_step)
             except (MachineError, KeyError, AttributeError):
@@ -239,7 +239,7 @@ def matching_cost_check_conf(self, cfg: Dict[str, dict], input_step: str) -> Non
         :return: None
         """
 
-        matching_cost_ = matching_cost.MatchingCost(**cfg[input_step])
+        matching_cost_ = matching_cost.MatchingCost(**cfg["pipeline"][input_step])
         self.pipeline_cfg["pipeline"][input_step] = matching_cost_.cfg
 
     def disparity_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None:
@@ -253,7 +253,7 @@ def disparity_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None:
         :return: None
         """
 
-        disparity_ = disparity.Disparity(**cfg[input_step])
+        disparity_ = disparity.Disparity(**cfg["pipeline"][input_step])
         self.pipeline_cfg["pipeline"][input_step] = disparity_.cfg
 
     def refinement_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None:
@@ -267,7 +267,7 @@ def refinement_check_conf(self, cfg: Dict[str, dict], input_step: str) -> None:
         :return: None
         """
 
-        refinement_ = refinement.AbstractRefinement(**cfg[input_step])  # type: ignore
+        refinement_ = refinement.AbstractRefinement(**cfg["pipeline"][input_step])  # type: ignore
         self.pipeline_cfg["pipeline"][input_step] = refinement_.cfg
 
     def matching_cost_run(self, cfg: Dict[str, dict], input_step: str) -> None:
@@ -282,7 +282,7 @@ def matching_cost_run(self, cfg: Dict[str, dict], input_step: str) -> None:
         """
 
         logging.info("Matching cost computation...")
-        matching_cost_run = matching_cost.MatchingCost(**cfg[input_step])
+        matching_cost_run = matching_cost.MatchingCost(**cfg["pipeline"][input_step])
 
         self.cost_volumes = matching_cost_run.compute_cost_volumes(
             self.left_img,
@@ -291,7 +291,7 @@ def matching_cost_run(self, cfg: Dict[str, dict], input_step: str) -> None:
             self.disp_max_col,
             self.disp_min_row,
             self.disp_max_row,
-            **cfg[input_step]
+            **cfg["pipeline"][input_step]
         )
 
     def disp_maps_run(self, cfg: Dict[str, dict], input_step: str) -> None:
@@ -306,7 +306,7 @@ def disp_maps_run(self, cfg: Dict[str, dict], input_step: str) -> None:
         """
 
         logging.info("Disparity computation...")
-        disparity_run = disparity.Disparity(**cfg[input_step])
+        disparity_run = disparity.Disparity(**cfg["pipeline"][input_step])
 
         map_col, map_row = disparity_run.compute_disp_maps(self.cost_volumes)
         self.dataset_disp_maps = common.dataset_disp_maps(map_row, map_col)
@@ -323,7 +323,7 @@ def refinement_run(self, cfg: Dict[str, dict], input_step: str) -> None:
         """
 
         logging.info("Refinement computation...")
-        refinement_run = refinement.AbstractRefinement(**cfg[input_step]) # type: ignore
+        refinement_run = refinement.AbstractRefinement(**cfg["pipeline"][input_step])  # type: ignore
 
         refine_map_col, refine_map_row = refinement_run.refinement_method(self.cost_volumes, self.dataset_disp_maps)
         self.dataset_disp_maps = common.dataset_disp_maps(refine_map_row, refine_map_col)
diff --git a/tests/common.py b/tests/common.py
@@ -61,19 +61,25 @@
 }
 
 correct_pipeline = {
-    "matching_cost": {"matching_cost_method": "zncc", "window_size": 5},
-    "disparity": {"disparity_method": "wta", "invalid_disparity": -99},
-    "refinement": {"refinement_method": "interpolation"},
+    "pipeline": {
+        "matching_cost": {"matching_cost_method": "zncc", "window_size": 5},
+        "disparity": {"disparity_method": "wta", "invalid_disparity": -99},
+        "refinement": {"refinement_method": "interpolation"}
+    }
 }
 
 false_pipeline_mc = {
-    "disparity": {"disparity_method": "wta", "invalid_disparity": -99},
-    "refinement": {"refinement_method": "interpolation"},
+    "pipeline": {
+        "disparity": {"disparity_method": "wta", "invalid_disparity": -99},
+        "refinement": {"refinement_method": "interpolation"}
+    }
 }
 
 false_pipeline_disp = {
-    "matching_cost": {"matching_cost_method": "zncc", "window_size": 5},
-    "refinement": {"refinement_method": "interpolation"},
+    "pipeline": {
+        "matching_cost": {"matching_cost_method": "zncc", "window_size": 5},
+        "refinement": {"refinement_method": "interpolation"}
+    }
 }
 
 correct_pipeline_dict = {

diff --git a/tests/test_pandora2d.py b/tests/test_pandora2d.py
@@ -61,8 +61,8 @@ def test_run_pandora() -> None:
         false_cfg_mc = copy.deepcopy(common.false_pipeline_mc)
         false_cfg_disp = copy.deepcopy(common.false_pipeline_disp)
         with pytest.raises(MachineError):
-            pandora2d_machine.check_conf(false_cfg_mc) # type: ignore
-            pandora2d_machine.check_conf(false_cfg_disp) # type: ignore
+            pandora2d_machine.check_conf(false_cfg_mc)  # type: ignore
+            pandora2d_machine.check_conf(false_cfg_disp)  # type: ignore
 
     @staticmethod
     def test_run_prepare() -> None: