From 6fba8029a5b9fef17ccbfdca7186cd98377f8659 Mon Sep 17 00:00:00 2001 From: janosmurai Date: Tue, 12 Nov 2024 15:30:08 +0100 Subject: [PATCH] Introduce option to use the system cert store for TLS authentication. Enhance configuration management by adding default options and loading logic for registries, catalogs, hosts, certs and timeout settings. Introduce platform configuration method and update documentation and test cases. --- dem/__main__.py | 3 + dem/core/data_management.py | 71 +++++++++++++++------- dem/core/platform.py | 7 ++- docs/configuration.md | 94 ++++++++++++++++++++++++++++++ mkdocs.yml | 3 +- pyproject.toml | 3 +- tests/core/test_data_management.py | 33 ++--------- 7 files changed, 162 insertions(+), 52 deletions(-) create mode 100644 docs/configuration.md diff --git a/dem/__main__.py b/dem/__main__.py index dfb44b6..c837683 100644 --- a/dem/__main__.py +++ b/dem/__main__.py @@ -25,6 +25,9 @@ def main() -> None: # Load the configuration file dem.cli.main.platform.config_file.update() + # Configure the Development Platform + dem.cli.main.platform.configure() + # Load the Dev Env descriptors dem.cli.main.platform.load_dev_envs() diff --git a/dem/core/data_management.py b/dem/core/data_management.py index f6a1515..8957c1c 100644 --- a/dem/core/data_management.py +++ b/dem/core/data_management.py @@ -89,23 +89,25 @@ class ConfigFile(BaseJSON): def __init__(self) -> None: """ Init the class.""" self._path = PurePath(self._config_dir + "/config.json") - self._default_json = """{ - "registries": [ - { - "name": "axem", - "namespace": "axemsolutions", - "url": "https://registry.hub.docker.com" - } - ], - "catalogs": [ - { - "name": "axem", - "url": "https://axemsolutions.io/dem/dev_env_org.json" + self._default_options = { + "registries": [ + { + "name": "axem", + "namespace": "axemsolutions", + "url": "https://registry.hub.docker.com" + } + ], + "catalogs": [ + { + "name": "axem", + "url": "https://axemsolutions.io/dem/dev_env_org.json" + } + ], + "hosts": [], + "http_request_timeout_s": 2, + "use_native_system_cert_store": False } - ], - "hosts": [], - "http_request_timeout_s": 2 -}""" + self._default_json = json.dumps(self._default_options, indent=4) super().__init__() def update(self) -> None: @@ -114,10 +116,37 @@ def update(self) -> None: except json.decoder.JSONDecodeError as e: raise DataStorageError(f"The config.json file is corrupted.\n{str(e)}") from e - self.registries: list[dict] = self.deserialized.get("registries", []) - self.catalogs: list[dict] = self.deserialized.get("catalogs", []) - self.hosts: list[dict] = self.deserialized.get("hosts", []) + flush_needed = False + + self.registries: list[dict] | None = self.deserialized.get("registries", None) + if self.registries is None: + self.deserialized["registries"] = self._default_options["registries"] + self.registries = self._default_options["registries"] + flush_needed = True + + self.catalogs: list[dict] | None = self.deserialized.get("catalogs", None) + if self.catalogs is None: + self.deserialized["catalogs"] = self._default_options["catalogs"] + self.catalogs = self._default_options["catalogs"] + flush_needed = True + + self.hosts: list[dict] | None = self.deserialized.get("hosts", None) + if self.hosts is None: + self.deserialized["hosts"] = self._default_options["hosts"] + self.hosts = self._default_options["hosts"] + flush_needed = True + self.http_request_timeout_s: float = self.deserialized.get("http_request_timeout_s", None) - if self.http_request_timeout_s is None: - raise DataStorageError("The http_request_timeout_s is not set in the config.json file.") \ No newline at end of file + self.deserialized["http_request_timeout_s"] = self._default_options["http_request_timeout_s"] + self.http_request_timeout_s = self._default_options["http_request_timeout_s"] + flush_needed = True + + self.use_native_system_cert_store: bool | None = self.deserialized.get("use_native_system_cert_store", None) + if self.use_native_system_cert_store is None: + self.deserialized["use_native_system_cert_store"] = self._default_options["use_native_system_cert_store"] + self.use_native_system_cert_store = self._default_options["use_native_system_cert_store"] + flush_needed = True + + if flush_needed: + self.flush() \ No newline at end of file diff --git a/dem/core/platform.py b/dem/core/platform.py index de1838d..2e08312 100644 --- a/dem/core/platform.py +++ b/dem/core/platform.py @@ -1,7 +1,7 @@ """Repesents the Development Platform. The platform resources can be accessed through this interface. """ -import os +import os, truststore from typing import Any, Generator from dem.core.core import Core from dem.core.properties import __supported_dev_env_major_version__ @@ -45,6 +45,11 @@ def __init__(self) -> None: # Set this to true in the platform instance to get the tool image info from the registries self.get_tool_image_info_from_registries = False + def configure(self) -> None: + """ Configure the Development Platform.""" + if self.config_file.use_native_system_cert_store: + truststore.inject_into_ssl() + def load_dev_envs(self) -> None: """ Load the Development Environments from the dev_env.json file. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..a13c9a2 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,94 @@ +## registries + +The `registries` section of the configuration file is used to define the registries that DEM will +use to search for images. + +**Default value:** + +```json +"registries": [ + { + "name": "axem", + "namespace": "axemsolutions", + "url": "https://registry.hub.docker.com" + } +] +``` + +!!! info + The Registry Management commands can be used to manage the registries in the configuration file. + + - [`add-reg`](commands.md#dem-add-reg-name-url-namespace): Add a new registry to the configuration file. + - [`del-reg`](commands.md#dem-del-reg-name): Delete a registry from the configuration file. + - [`list-reg`](commands.md#dem-list-reg): List the registries in the configuration file. + + +## catalogs + +The `catalogs` section of the configuration file is used to define the catalogs that DEM will use to +search for Development Environment descriptors. + +** Default value: ** + +```json +"catalogs": [ + { + "name": "axem", + "url": "https://axemsolutions.io/dem/dev_env_org.json" + } +] +``` + +!!! info + The Catalog Management commands can be used to manage the catalogs in the configuration file. + + - [`add-cat`](commands.md#dem-add-cat-name-url): Add a new catalog to the configuration file. + - [`del-cat`](commands.md#dem-del-cat-name): Delete a catalog from the configuration file. + - [`list-cat`](commands.md#dem-list-cat): List the catalogs in the configuration file. + +## hosts + +!!! warning + Remote hosts are not yet supported in DEM. + +The `hosts` section of the configuration file is used to define the hosts that DEM will use as +remote execution environments. + +**Default value:** + +```json +"hosts": [] +``` + +!!! info + The Host Management commands can be used to manage the hosts in the configuration file. + + - [`add-host`](commands.md#dem-add-host-name-url): Add a new host to the configuration file. + - [`del-host`](commands.md#dem-del-host-name): Delete a host from the configuration file. + - [`list-host`](commands.md#dem-list-host): List the hosts in the configuration file. + +## http_request_timeout_s + +The `http_request_timeout_s` section of the configuration file is used to define the timeout for +HTTP requests in seconds. + +**Default value:** + +```json +"http_request_timeout_s": 2 +``` + +## use_native_system_cert_store + +The `use_native_system_cert_store` section of the configuration file is used to define whether +the native system certificate store should be used for HTTPS requests or the default one provided +by the `certifi` package. + +**Default value:** + +```json +"use_native_system_cert_store": false +``` + +!!! info + If the TLS authentication fails, try setting this value to `true`. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index f183603..2a50bec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -92,6 +92,7 @@ nav: - 'index.md' - 'installation.md' - 'basics.md' + - 'quickstart.md' - Commands: - 'DevEnv Management': - 'add-task': 'commands/#dem-add-task-dev_env_name-task_name-command' @@ -125,6 +126,6 @@ nav: - 'add-host': 'commands/#dem-add-host-name-address' - 'del-host': 'commands/#dem-del-host-name' - 'list-host': 'commands/#dem-list-host' - - 'quickstart.md' + - 'configuration.md' - 'design.md' - Join us on Discord : 'https://discord.gg/3aHuJBNvrJ' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5b77862..f9857f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,10 @@ dem = "dem.__main__:main" python = "^3.10" docker = "^7.0.0" readchar = "^4.0.6" -requests = "^2.31.0" +requests = "2.31.0" rich = "^13.7.1" typer = "^0.12.2" +truststore = "^0.10.0" [tool.poetry.group.docs] optional = true diff --git a/tests/core/test_data_management.py b/tests/core/test_data_management.py index 17b4398..0504451 100644 --- a/tests/core/test_data_management.py +++ b/tests/core/test_data_management.py @@ -170,10 +170,6 @@ def test_ConfigFile(mock_PurePath: MagicMock): test_path = "test_path" data_management.BaseJSON._config_dir = test_path - mock_registries = MagicMock() - mock_catalogs = MagicMock() - mock_hosts = MagicMock() - # Run unit under test local_dev_env_json = data_management.ConfigFile() @@ -194,13 +190,15 @@ def test_ConfigFile(mock_PurePath: MagicMock): } ], "hosts": [], - "http_request_timeout_s": 2 + "http_request_timeout_s": 2, + "use_native_system_cert_store": false }""" mock_PurePath.assert_called_once_with(test_path + "/config.json") +@patch.object(data_management.BaseJSON, "flush") @patch.object(data_management.BaseJSON, "update") -def test_ConfigFile_update(mock_update: MagicMock) -> None: +def test_ConfigFile_update(mock_update: MagicMock, mock_flush: MagicMock) -> None: # Test setup test_config_file = data_management.ConfigFile() test_registry = MagicMock() @@ -224,28 +222,7 @@ def test_ConfigFile_update(mock_update: MagicMock) -> None: assert test_config_file.http_request_timeout_s == test_http_request_timeout_s mock_update.assert_called_once() - -@patch.object(data_management.BaseJSON, "update") -def test_ConfigFile_update_missing_http_request_timeout_s(mock_update: MagicMock) -> None: - # Test setup - test_config_file = data_management.ConfigFile() - test_registry = MagicMock() - test_catalog = MagicMock() - test_host = MagicMock() - test_config_file.deserialized = { - "registries": [test_registry], - "catalogs": [test_catalog], - "hosts": [test_host], - } - - with pytest.raises(data_management.DataStorageError) as e: - # Run unit under test - test_config_file.update() - - # Check expectations - assert "Invalid file: The http_request_timeout_s is not set in the config.json file." == str(e.value) - - mock_update.assert_called_once() + mock_flush.assert_called_once() @patch.object(data_management.BaseJSON, "update") def test_ConfigFile_update_JSONDecodeError(mock_update: MagicMock) -> None: