Allow user templates for generation during terraform lifecycle (#64)

Templates can now be added to an infrastructure file under the `templates` key, which is a list of template file names, and it will be generated after all resources have been created at the root directory. Primarily needed for generating inventory files for ansible. Tags are recommended to assist in most templating as other outputs might vary by provider and/or service. * Templates * To overcome the limitations of `templatefile()` not allowing for a new file to be referenced in the same run, we copy all templates into `<project_path>/templates/` and update the users template list to point to the new file with `template/<templatefile>`. * generated files stored in the project root directory * Outputs * `local.outputs` created and reused for our various outputs * per server type outputs were removed since it requires jinja2 * `servers.yml` contains the same values that will generate the templates * Logs updated to allow for overrides with `ET_` and displayed under help
EnterpriseDB · May 25, 2023 · 3040a23 · 3040a23
1 parent e31ad92
commit 3040a23
Show file tree

Hide file tree

Showing 16 changed files with 322 additions and 160 deletions.
diff --git a/README.md b/README.md
@@ -182,27 +182,37 @@ resources creation:
 
 Once cloud resources provisioning is completed, machines public and private IPs
 are stored in the `servers.yml` file, located into the project's directory.
+These outputs can be used with a list of templates to generate files for other programs such as ansible.
+See example here which uses the below outputs. [inventory.yml](./infrastructure-examples/v2_inventory.yml.tftpl)
 
 Example:
 
 ```yaml
 ---
 servers:
-  barman1:
-    type: barman
-    region: us-east-1
-    zone: us-east-1b
-    public_ip: 54.166.46.2
-    private_ip: 10.0.0.103
-    # Default provided DNS only supported by AWS
-    public_dns: ec2-54-166-46-2.compute-1.amazonaws.com
-  pg1:
-    type: postgres
-    region: us-east-1
-    zone: us-east-1b
-    public_ip: 3.80.202.134
-    private_ip: 10.0.0.148
-    public_dns: ec2-3-80-202-134.compute-1.amazonaws.com
+  machines:
+    dbt2-driver:
+      additional_volumes: []
+      instance_type: "c5.4xlarge"
+      operating_system: {"name":"debian-10-amd64","owner":"136693071363","ssh_user":"admin"}
+      private_ip: "10.2.20.38"
+      public_dns: "ec2-54-197-78-139.compute-1.amazonaws.com"
+      public_ip: "54.197.78.139"
+      region: "us-east-1"
+      tags: {"Name":"dbt2-driver-Demo-Infra-d8d0a932","cluster_name":"Demo-Infra","created_by":"edb-terraform","terraform_hex":"d8d0a932","terraform_id":"2NCpMg","terraform_time":"2023-05-24T21:09:11Z","type":"dbt2-driver"}
+      type: null
+      zone: "us-east-1b"
+    pg1:
+      additional_volumes: [{"encrypted":false,"iops":5000,"mount_point":"/opt/pg_data","size_gb":20,"type":"io2"},{"encrypted":false,"iops":5000,"mount_point":"/opt/pg_wal","size_gb":20,"type":"io2"}]
+      instance_type: "c5.4xlarge"
+      operating_system: {"name":"Rocky-8-ec2-8.6-20220515.0.x86_64","owner":"679593333241","ssh_user":"rocky"}
+      private_ip: "10.2.30.197"
+      public_dns: "ec2-3-89-238-24.compute-1.amazonaws.com"
+      public_ip: "3.89.238.24"
+      region: "us-east-1"
+      tags: {"Name":"pg1-Demo-Infra-d8d0a932","cluster_name":"Demo-Infra","created_by":"edb-terraform","terraform_hex":"d8d0a932","terraform_id":"2NCpMg","terraform_time":"2023-05-24T21:09:11Z","type":"postgres"}
+      type: null
+      zone: "us-east-1b"
 [...]
 ```
 

diff --git a/edbterraform/CLI.py b/edbterraform/CLI.py
@@ -10,7 +10,7 @@
 import textwrap
 
 from edbterraform import __project_name__
-from edbterraform.Logger import logger
+from edbterraform.utils.logs import logger
 
 Version = namedtuple('Version', ['major', 'minor', 'patch'])
 

diff --git a/edbterraform/Logger.py b/edbterraform/Logger.py
diff --git a/edbterraform/args.py b/edbterraform/args.py
@@ -6,11 +6,12 @@
 from dataclasses import dataclass, field
 from collections import OrderedDict
 from typing import List
+from datetime import datetime
 
 from edbterraform.lib import generate_terraform
 from edbterraform.CLI import TerraformCLI
 from edbterraform import __project_name__
-from edbterraform.Logger import logger
+from edbterraform.utils import logs
 
 ENVIRONMENT_PREFIX = 'ET_' # Appended to allow overrides of defaults
 
@@ -152,6 +153,47 @@ def __getitem__(self, key):
         '''
 )
 
+LogLevel = ArgumentConfig(
+    names = ['--log_level',],
+    dest='log_level',
+    required=False,
+    default="INFO",
+    help='''
+        Default: %(default)s
+        '''
+)
+
+LogFile = ArgumentConfig(
+    names = ['--log_file',],
+    dest='log_file',
+    required=False,
+    default=datetime.now().strftime('%Y-%m-%d'),
+    help='''
+        Default: %(default)s
+        '''
+)
+
+LogDirectory = ArgumentConfig(
+    names = ['--log_directory',],
+    dest='log_directory',
+    required=False,
+    default=f'{Path.home()}/.{__project_name__}/logs',
+    help='''
+        Default: %(default)s
+        '''
+)
+
+LogStdout = ArgumentConfig(
+    names = ['--log_stdout',],
+    dest='log_stdout',
+    action='store_true',
+    required=False,
+    default=True,
+    help='''
+        Default: %(default)s
+        '''
+)
+
 class Arguments:
 
     # Command, description, and its options
@@ -162,6 +204,10 @@ class Arguments:
             CloudServiceProvider,
             Validation,
             BinPath,
+            LogLevel,
+            LogFile,
+            LogDirectory,
+            LogStdout,
         ]],
         'generate': ['Generate terraform files based on a yaml infrastructure file\n',[
             ProjectName,
@@ -170,9 +216,17 @@ class Arguments:
             CloudServiceProvider,
             Validation,
             BinPath,
+            LogLevel,
+            LogFile,
+            LogDirectory,
+            LogStdout,
         ]],
         'setup': ['Install needed software such as Terraform inside a bin directory\n',[
             BinPath,
+            LogLevel,
+            LogFile,
+            LogDirectory,
+            LogStdout,
         ]],
     })
     DEFAULT_COMMAND = next(iter(COMMANDS))
@@ -235,6 +289,12 @@ def get_env(self, key, default=None):
         return getattr(self.env, key, default)
 
     def process_args(self):
+        logs.setup_logs(
+            level=self.get_env('log_level'),
+            file_name=self.get_env('log_file'),
+            directory=self.get_env('log_directory'),
+            stdout=self.get_env('log_stdout'),
+        )
         if self.command == 'depreciated':
             outputs = generate_terraform(
                 self.get_env('infra_file'),

diff --git a/edbterraform/data/templates/aws/main.tf.j2 b/edbterraform/data/templates/aws/main.tf.j2
@@ -38,12 +38,6 @@
 {%   include "region_peering.tf.j2" %}
 {% endif %}
 
-resource "local_file" "servers_yml" {
-  filename        = "${abspath(path.root)}/servers.yml"
-  file_permission = "0600"
-  content         = <<-EOT
----
-servers:
 {% set boxes = {
   'machines': { 
     'active': has_machines,
@@ -71,37 +65,48 @@ servers:
     'module_base': "module.kubernetes_",
   }  
 } %}
+locals {
+  # outputs set with the help of jinja2 in edb-terraform
+  outputs = {
 {% for type, attributes in boxes.items() if attributes["active"] %}
-  {{type}}:
+    "{{type}}" = merge(
 {%   for region in attributes["regions"] -%}
 {%   set module = attributes["module_base"] ~ region | replace('-', '_') %}
-%{ for key, value in {{ module }} ~}
-    ${key}:
-%{   for name, item in value ~}
-      ${name}: ${try(jsonencode(item), "Error, unsupported type",)}
-%{   endfor ~}
-%{ endfor ~}
+    tomap([for key, values in {{module}}[*]: values]...),
 {%   endfor %}
+    )
 {% endfor %}
-    EOT
+  }
+  servers = { "servers" = local.outputs }
 }
 
-{% for type, attributes in boxes.items() if attributes["active"] %}
-output "{{type}}" {
-  value = [
-{%   for region in attributes["regions"] -%}
-{%   set module = attributes["module_base"] ~ region | replace('-', '_') %}
-    {{ module }}[*],
-{%   endfor %}
-  ]
-  sensitive = true
+resource "local_file" "servers_yml" {
+  /*
+  Manually create the yaml
+  TODO: Update to yamlencode function once stable
+  */
+  filename        = "${abspath(path.root)}/servers.yml"
+  file_permission = "0600"
+  content         = <<-EOT
+---
+servers:
+%{ for type, instances in local.outputs ~}
+    ${type}:
+%{   for name, attributes in instances ~}
+      ${name}:
+%{     for key, values in attributes ~}
+        ${key}: ${jsonencode(values)}
+%{     endfor ~}
+%{   endfor ~}
+%{ endfor ~}
+    EOT
 }
-{% endfor %}
 
 output "{{output_name}}" {
   description = <<-EOT
+  toplevel default made through jinja2 templating with edb-terraform: set to servers
   Use 'terraform output -json' for the following output and other info such as types:
-  servers:
+  {{output_name}}:
     value:
       machines:
         machine_name:
@@ -118,20 +123,23 @@ output "{{output_name}}" {
     database_name:
       instance_type: <instance_type>
   EOT
-  value = {
-{% for type, attributes in boxes.items() if attributes["active"] %}
-    "{{type}}" = merge(
-{%   for region in attributes["regions"] -%}
-{%   set module = attributes["module_base"] ~ region | replace('-', '_') %}
-    tomap([for key, values in {{module}}[*]: values]...),
-{%   endfor %}
-    )
-{% endfor %}
-  }
+  value = local.outputs
   sensitive = true
 }
 
 output "spec" {
   value = module.spec[*]
   sensitive = true
 }
+
+resource "local_file" "user_templates" {
+  /*
+  User custom templates with local.outputs passed in for generation
+    output_name default made through jinja2 templating with edb-terraform: 'servers'
+    terraform output -json <output_name>
+  */
+  for_each   = toset(module.spec.base.templates)
+  content    = templatefile(each.value, local.servers)
+  filename   = "${abspath(path.root)}/${trimsuffix(basename(each.value), ".tftpl")}"
+  file_permission = "0600"
+}