From 9c7ad707ce0631f80b1374e03840339e1c39fe7a Mon Sep 17 00:00:00 2001 From: Simon Westphahl Date: Tue, 19 May 2020 19:45:02 +0200 Subject: [PATCH 1/4] Refactor internal context handling YAML instances that are responsible for en-/decrypting all live in the 'vault' module now and no longer in 'context'. This way we can also dump plain YAML from konverter-vault commands without causing circular imports. --- src/konverter/context.py | 48 ++++------------------ src/konverter/vault.py | 88 +++++++++++++++++++++++++++++++--------- 2 files changed, 75 insertions(+), 61 deletions(-) diff --git a/src/konverter/context.py b/src/konverter/context.py index 0e99086..b9daaaf 100644 --- a/src/konverter/context.py +++ b/src/konverter/context.py @@ -1,49 +1,12 @@ from __future__ import annotations +import functools import pathlib import typing from cryptography.fernet import Fernet -from ruamel.yaml.nodes import ScalarNode -from .vault import KonvertEncrypt, KonvertVault, key_from_file -from .yaml import BaseYAML - -if typing.TYPE_CHECKING: - from ruamel.yaml.constructor import Constructor - - -class KonvertContextVault(KonvertVault): - @classmethod - def from_yaml(cls, constructor: Constructor, node: ScalarNode, yaml) -> ScalarNode: - value = yaml.fernet.decrypt(bytes(node.value, encoding="utf8")) - return constructor.construct_scalar( - ScalarNode( - tag="tag:yaml.org,2002:str", - value=str(value, encoding="utf8"), - style=node.style, - ) - ) - - -class KonvertContextEncrypt(KonvertEncrypt): - @classmethod - def from_yaml(cls, constructor: Constructor, node: ScalarNode, yaml) -> ScalarNode: - return constructor.construct_scalar( - ScalarNode(tag="tag:yaml.org,2002:str", value=node.value, style=node.style) - ) - - -class ContextYAML(BaseYAML): - def __init__(self, provider: ContextProvider): - super().__init__() - self.provider = provider - self.register_type(KonvertContextEncrypt) - self.register_type(KonvertContextVault) - - @property - def fernet(self) -> Fernet: - return self.provider.fernet +from .vault import key_from_file, VaultToPlainYAML class ContextProvider: @@ -54,7 +17,6 @@ def __init__( self.key_path = key_path self._fernet: typing.Optional[Fernet] = None - @property def fernet(self) -> Fernet: if self._fernet is None: self._fernet = Fernet(key_from_file(self.work_dir / self.key_path)) @@ -62,4 +24,8 @@ def fernet(self) -> Fernet: def load_context(self, path: pathlib.Path) -> typing.Mapping[str, typing.Any]: with open(path) as yaml_file: - return ContextYAML(self).load(yaml_file) + # We don't pass the property directly as this would try to load the + # key from file which won't work when no key file is provided. + # This is a valid use case e.g. when the context doesn't use + # encrypted values. + return VaultToPlainYAML(functools.partial(self.fernet)).load(yaml_file) diff --git a/src/konverter/vault.py b/src/konverter/vault.py index d1de877..a69f329 100644 --- a/src/konverter/vault.py +++ b/src/konverter/vault.py @@ -7,11 +7,12 @@ import click from cryptography.fernet import Fernet +from ruamel.yaml.nodes import ScalarNode from .yaml import BaseYAML, KonvertType if typing.TYPE_CHECKING: - from ruamel.yaml.nodes import ScalarNode + from ruamel.yaml.constructor import Constructor from ruamel.yaml.representer import Representer DEFAULT_VAULT_KEY_PATH = ".konverter-vault" @@ -47,30 +48,77 @@ def to_yaml( ) -class DecryptYAML(BaseYAML): - def __init__(self, fernet: Fernet): +class KonvertVaultValue(KonvertVault): + @classmethod + def from_yaml(cls, constructor: Constructor, node: ScalarNode, yaml) -> ScalarNode: + value = yaml.fernet.decrypt(bytes(node.value, encoding="utf8")) + return constructor.construct_scalar( + ScalarNode( + tag="tag:yaml.org,2002:str", + value=str(value, encoding="utf8"), + style=node.style, + ) + ) + + +class KonvertEncryptValue(KonvertEncrypt): + @classmethod + def from_yaml(cls, constructor: Constructor, node: ScalarNode, yaml) -> ScalarNode: + return constructor.construct_scalar( + ScalarNode(tag="tag:yaml.org,2002:str", value=node.value, style=node.style) + ) + + +class VaultYAML(BaseYAML): + def __init__(self, fernet: typing.Callable[[], Fernet]): super().__init__() - self.fernet = fernet + if isinstance(fernet, Fernet): + self.fernet: Fernet = fernet + else: + self._lazy_fernet = fernet + + def __getattr__(self, name): + if name == "fernet": + self.fernet = self._lazy_fernet() + return self.fernet + raise AttributeError(name) + + +class VaultToEditableYAML(VaultYAML): + def __init__(self, fernet: typing.Callable[[], Fernet]): + super().__init__(fernet) self.register_type(KonvertVault) - def decrypt(self, encrypted: typing.IO[str], decrypted: typing.IO[str]) -> None: - data = self.load(encrypted) + def convert(self, vault: typing.IO[str], decrypted: typing.IO[str]) -> None: + data = self.load(vault) if data is None: return self.dump(data, stream=decrypted) -class EncryptYAML(BaseYAML): - def __init__(self, fernet: Fernet): - super().__init__() - self.fernet = fernet +class EditableToVaultYAML(VaultYAML): + def __init__(self, fernet: typing.Callable[[], Fernet]): + super().__init__(fernet) self.register_type(KonvertEncrypt) - def encrypt(self, decrypted: typing.IO[str], encrypted: typing.IO[str]) -> None: + def convert(self, decrypted: typing.IO[str], vault: typing.IO[str]) -> None: data = self.load(decrypted) if data is None: return - self.dump(data, stream=encrypted) + self.dump(data, stream=vault) + + +class VaultToPlainYAML(VaultYAML): + def __init__(self, fernet: typing.Callable[[], Fernet]): + super().__init__(fernet) + self.register_type(KonvertEncryptValue) + self.register_type(KonvertVaultValue) + + def convert(self, encrypted: typing.IO[str], plain: typing.IO[str]) -> None: + data = self.load(encrypted) + if data is None: + return + self.dump(data, stream=plain) def key_from_file(key_path: typing.Union[str, pathlib.Path]) -> bytes: @@ -112,10 +160,10 @@ def keygen(ctx: click.Context) -> None: def encrypt(ctx: click.Context, file_path: str): fernet = ctx.obj["fernet"] - encrypt_yaml = EncryptYAML(fernet) + encrypt_yaml = EditableToVaultYAML(fernet) with tempfile.NamedTemporaryFile(mode="wt", suffix=".yaml") as tmp_file: with open(file_path, "r") as yaml_file: - encrypt_yaml.encrypt(yaml_file, tmp_file) + encrypt_yaml.convert(yaml_file, tmp_file) tmp_file.flush() shutil.copy(tmp_file.name, file_path) @@ -127,10 +175,10 @@ def encrypt(ctx: click.Context, file_path: str): def decrypt(ctx: click.Context, file_path: str): fernet = ctx.obj["fernet"] - decrypt_yaml = DecryptYAML(fernet) + decrypt_yaml = VaultToEditableYAML(fernet) with tempfile.NamedTemporaryFile(mode="wt", suffix=".yaml") as tmp_file: with open(file_path, "r") as yaml_file: - decrypt_yaml.decrypt(yaml_file, tmp_file) + decrypt_yaml.convert(yaml_file, tmp_file) tmp_file.flush() shutil.copy(tmp_file.name, file_path) @@ -142,10 +190,10 @@ def decrypt(ctx: click.Context, file_path: str): def edit(ctx: click.Context, file_path: str): fernet = ctx.obj["fernet"] - decrypt_yaml = DecryptYAML(fernet) + decrypt_yaml = VaultToEditableYAML(fernet) with tempfile.NamedTemporaryFile(mode="w+t", suffix=".yaml") as tmp_file: with open(file_path, "r+") as yaml_file: - decrypt_yaml.decrypt(yaml_file, tmp_file) + decrypt_yaml.convert(yaml_file, tmp_file) tmp_file.flush() while True: @@ -153,9 +201,9 @@ def edit(ctx: click.Context, file_path: str): tmp_file.flush() tmp_file.seek(0) try: - encrypt_yaml = EncryptYAML(fernet) + encrypt_yaml = EditableToVaultYAML(fernet) with tempfile.NamedTemporaryFile(mode="wt") as out_tmp: - encrypt_yaml.encrypt(tmp_file, out_tmp) + encrypt_yaml.convert(tmp_file, out_tmp) out_tmp.flush() shutil.copy(out_tmp.name, file_path) break From c8e0f10432b2431005771ba0b93a53ec236aeef6 Mon Sep 17 00:00:00 2001 From: Simon Westphahl Date: Tue, 19 May 2020 20:44:06 +0200 Subject: [PATCH 2/4] Add command to show an unencrypted vault file The new 'konverter-vault show' command supports YAML and JSON as output format. --- CHANGELOG.md | 2 ++ src/konverter/vault.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 004bfdd..91a0ce2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +- Added a `konverter-vault show` command that supports YAML and JSON output + ## [v0.2.0] - 2020-03-01 - Added support for directories as context path diff --git a/src/konverter/vault.py b/src/konverter/vault.py index a69f329..8a4839e 100644 --- a/src/konverter/vault.py +++ b/src/konverter/vault.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import pathlib import shutil import tempfile @@ -217,5 +218,28 @@ def edit(ctx: click.Context, file_path: str): ) +@cli.command() +@click.argument("file_path", type=click.Path(exists=True, dir_okay=False)) +@click.option( + "-f", + "--format", + "output_format", + type=click.Choice(["yaml", "json"], case_sensitive=False), + default="yaml", +) +@click.pass_context +def show(ctx: click.Context, file_path: str, output_format: str): + fernet = ctx.obj["fernet"] + + encrypt_yaml = VaultToPlainYAML(fernet) + with open(file_path, "r+") as yaml_file: + with click.open_file("-", "wt") as stdout: + data = encrypt_yaml.load(yaml_file) + if output_format == "json": + json.dump(data, stdout, indent=2) + else: + encrypt_yaml.dump(data, stdout) + + if __name__ == "__main__": cli() From 8797503de64b6782db8ff2063d4b37961eb8a0f4 Mon Sep 17 00:00:00 2001 From: Simon Westphahl Date: Tue, 19 May 2020 21:01:51 +0200 Subject: [PATCH 3/4] Add terraform output format to vault show command With the terraform output format the 'konverter-vault show' command can be used as an external data source in Terraform. --- CHANGELOG.md | 3 +++ src/konverter/vault.py | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a0ce2..6e2b63f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## [Unreleased] - Added a `konverter-vault show` command that supports YAML and JSON output +- Added support for `terraform` output format in `konverter-vault show` command + that allows using it as an [external data source in + Terraform](https://www.terraform.io/docs/providers/external/data_source.html). ## [v0.2.0] - 2020-03-01 diff --git a/src/konverter/vault.py b/src/konverter/vault.py index 8a4839e..d76cf14 100644 --- a/src/konverter/vault.py +++ b/src/konverter/vault.py @@ -218,13 +218,24 @@ def edit(ctx: click.Context, file_path: str): ) +def to_terraform_format(data: dict) -> dict: + def _convert(v): + if isinstance(v, str): + return v + if isinstance(v, (float, int, bool)): + return str(v) + raise TypeError("Terraform JSON output only supports string values") + + return {k: _convert(v) for k, v in data.items() if v is not None} + + @cli.command() @click.argument("file_path", type=click.Path(exists=True, dir_okay=False)) @click.option( "-f", "--format", "output_format", - type=click.Choice(["yaml", "json"], case_sensitive=False), + type=click.Choice(["yaml", "json", "terraform"], case_sensitive=False), default="yaml", ) @click.pass_context @@ -237,6 +248,8 @@ def show(ctx: click.Context, file_path: str, output_format: str): data = encrypt_yaml.load(yaml_file) if output_format == "json": json.dump(data, stdout, indent=2) + elif output_format == "terraform": + json.dump(to_terraform_format(data), stdout, indent=2) else: encrypt_yaml.dump(data, stdout) From 56978fa3ee955d1d3c6d47f903dbcec0c3ebecb2 Mon Sep 17 00:00:00 2001 From: Simon Westphahl Date: Tue, 19 May 2020 21:11:51 +0200 Subject: [PATCH 4/4] Document new 'konverter-vault show' command --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index fd6b925..3382253 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,40 @@ $ konverter-vault encrypt vars/secret.yaml $ konverter-vault decrypt vars/secret.yaml ``` +In case the decrypted content should be passed to another tool that expects +YAML or JSON, the `konverter-vault show` command can come in handy: + +```shell +$ konverter-vault show --format=json vars/secret.yaml +{ + "credentials": { + "user": "root", + "password": "secret-password" + } +} +``` + +It will decrypt the given file and remove all `!k/(encrypt|vault)` YAML tags. +Supported output formats are `yaml` (default), `json` or `terraform`. + +The `terraform` output format is usefull for using `konverter-vault` as an +[external data source in +Terraform](https://www.terraform.io/docs/providers/external/data_source.html) + +```hcl +data "external" "konverter" { + program = [ + "konverter-vault", "show", "--format=terraform", "vars/secrets.yaml" + ] +} +``` + +Unfortunately the "[external program +protocol](https://www.terraform.io/docs/providers/external/data_source.html#external-program-protocol)" +only allows string values (no lists or objects). All `float`, `int` and `bool` +values will be converted to strings. Other types will cause an error when +using this output format. + ### Advanced configuration The above config file could be rewritten in a more verbose format: