From 8bf5f4f654f7a8229c889bf92f9d38770c81d8c8 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 19 Dec 2024 13:59:54 +0100 Subject: [PATCH 01/36] feat: make pre-commit scripts versioned --- infra/scripts/lock-modules.sh | 5 ++--- infra/scripts/package.json | 6 ++++++ package.json | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 infra/scripts/package.json diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 5d60df07..9a6df45f 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -26,7 +26,6 @@ for TARGET_DIR in "$@"; do if [ -d "$TARGET_DIR" ]; then echo "Processing Terraform modules in $TARGET_DIR" cd $TARGET_DIR - terraform init >&2 # Check if hashes file already exists; otherwise, create it if [ ! -f "$HASHES_FILE" ]; then @@ -35,8 +34,8 @@ for TARGET_DIR in "$@"; do # Check if modules metadata exists if [ ! -f "$MODULES_METADATA" ]; then - echo "Modules metadata file not found. Ensure that 'terraform init' has been run." >&2 - exit 1 + echo "Modules metadata file not found. I am going to execute a 'terraform init'." >&2 + terraform init >&2 fi # Iterate over modules listed in the metadata that were sourced from the Terraform registry diff --git a/infra/scripts/package.json b/infra/scripts/package.json new file mode 100644 index 00000000..9befb0fa --- /dev/null +++ b/infra/scripts/package.json @@ -0,0 +1,6 @@ +{ + "name": "pre_commit_scripts", + "version": "0.0.0", + "private": true +} + \ No newline at end of file diff --git a/package.json b/package.json index 155d4971..4187b5f3 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "workspaces": [ "packages/**", "website", - "infra/modules/*" + "infra/modules/*", + "infra/scripts" ], "devDependencies": { "prettier": "3.2.5", From 92a7a97a87474f7dbfd9111a427b3a200ad0571d Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 15:21:12 +0100 Subject: [PATCH 02/36] refactor: pre-commit modules lock --- .pre-commit-config.yaml | 8 ++ infra/scripts/lock-modules.sh | 259 +++++++++++++++++++++++++++------- 2 files changed, 214 insertions(+), 53 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b7ac411..5a6ab950 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,12 @@ repos: + - repo: local + hooks: + - id: terraform-modules-lock + name: Lock Terraform Modules + entry: infra/scripts/lock-modules.sh + language: script + files: '\.tf$' + pass_filenames: false - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.96.2 hooks: diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 9a6df45f..5beb43a2 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -1,72 +1,225 @@ #!/bin/bash set -euo pipefail -# Paths and files -MODULES_DIR=".terraform/modules" -HASHES_FILE="tfmodules.lock.json" -MODULES_METADATA=".terraform/modules/modules.json" -REGISTRY_URL="registry.terraform.io" +# Constants +readonly MODULES_DIR=".terraform/modules" +readonly HASHES_FILE="tfmodules.lock.json" +readonly MODULES_METADATA=".terraform/modules/modules.json" +readonly REGISTRY_URL="registry.terraform.io" +readonly SCRIPT_NAME=$(basename "$0") + +# Helper Functions +log_info() { + echo "[${SCRIPT_NAME}] INFO: $1" +} + +log_error() { + echo "[${SCRIPT_NAME}] ERROR: $1" >&2 +} + +log_warning() { + echo "[${SCRIPT_NAME}] WARNING: $1" >&2 +} + +find_terraform_dirs() { + local base_dir="$1" + # Find directories containing .tf files, excluding .terraform directories + find "$base_dir" -type f -name "*.tf" ! -path "*/.terraform/*" -exec dirname {} \; | sort -u +} calculate_hash() { local module_path="$1" - tar --exclude=.* -cvf - "$module_path" | sha256sum | awk '{ print $1 }' + tar --exclude=.* -czf - "$module_path" 2>/dev/null | sha256sum | awk '{ print $1 }' } -# If no arguments are passed, set a default value (e.g., current directory) -if [ "$#" -eq 0 ]; then - echo "No directories specified. Exiting." - TARGET_DIR="$PWD" -fi +init_hashes_file() { + local hashes_file="$1" + if [ ! -f "$hashes_file" ]; then + log_info "Creating new hashes file" + echo "{}" > "$hashes_file" + fi +} -BASE_DIR="$PWD" +ensure_terraform_init() { + # if [ ! -f "$MODULES_METADATA" ]; then + log_warning "Running terraform init in $(pwd)" + terraform init -input=false >/dev/null + # fi +} -# Iterate over all provided directories -for TARGET_DIR in "$@"; do - # Add your logic here to handle each directory - if [ -d "$TARGET_DIR" ]; then - echo "Processing Terraform modules in $TARGET_DIR" - cd $TARGET_DIR +process_module() { + local module_path="$1" + local module_name=$(basename "$module_path") + local new_hash=$(calculate_hash "$module_path") + local previous_hash + + # Retrieve the previous hash + previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "$HASHES_FILE") - # Check if hashes file already exists; otherwise, create it - if [ ! -f "$HASHES_FILE" ]; then - echo "{}" > "$HASHES_FILE" - fi + # Save the new hash + jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ + "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + + if [ "$previous_hash" = "none" ]; then + log_info "Module $module_name: Initial hash created" + return 1 + elif [ "$previous_hash" != "$new_hash" ]; then + log_info "Module $module_name: Changes detected, updating hash" + return 1 + else + log_info "Module $module_name: No changes detected" + return 0 + fi +} + +get_modules_from_tf_files() { + # Extract module sources from .tf files + grep -h 'source[[:space:]]*=' *.tf 2>/dev/null | \ + sed -E 's/.*source[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/' | \ + sort -u || echo "" +} + +get_modules_from_metadata() { + # Extract module sources from modules.json + if [ -f "$MODULES_METADATA" ]; then + jq -r '.Modules[].Source' "$MODULES_METADATA" 2>/dev/null | sort -u || echo "" + else + echo "" + fi +} + +needs_terraform_init() { + local current_modules + local metadata_modules + + # If modules.json doesn't exist, we need to init + if [ ! -f "$MODULES_METADATA" ]; then + log_info "No modules.json found, terraform init needed" + return 0 + fi + + # Get current modules from .tf files + current_modules=$(get_modules_from_tf_files) + if [ -z "$current_modules" ]; then + log_info "No modules found in .tf files" + return 1 + fi + + # Get modules from metadata + metadata_modules=$(get_modules_from_metadata) + + # Compare the sorted unique lists + if [ "$current_modules" != "$metadata_modules" ]; then + log_info "Module changes detected, terraform init needed" + log_info "Current modules:" + echo "$current_modules" + log_info "Cached modules:" + echo "$metadata_modules" + return 0 + fi - # Check if modules metadata exists - if [ ! -f "$MODULES_METADATA" ]; then - echo "Modules metadata file not found. I am going to execute a 'terraform init'." >&2 - terraform init >&2 + log_info "No module changes detected, skipping terraform init" + return 1 +} + +ensure_terraform_init() { + if needs_terraform_init; then + log_warning "Running terraform init in $(pwd)" + if ! terraform init -input=false >/dev/null; then + log_error "Terraform init failed" + return 1 fi + fi + return 0 +} - # Iterate over modules listed in the metadata that were sourced from the Terraform registry - jq -r --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA" | while read -r module_key; do - - module_path="$MODULES_DIR/$module_key" - - if [ -d "$module_path" ]; then - module_name=$(basename "$module_path") - new_hash=$(calculate_hash "$module_path") - - # Retrieve the previous hash - previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") - - # Save the new hash if not found in the file - jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' "$HASHES_FILE" > tmp.$$.json && mv tmp.$$.json "$HASHES_FILE" - echo "Saving the new hash for module $module_name." - # Compare the hashes - if [ "$previous_hash" == "$new_hash" ]; then - echo "The module $module_name has not changed." +process_directory() { + local target_dir="$1" + local base_dir="$2" + local changes_found=0 + + if [ ! -d "$target_dir" ]; then + log_error "Directory $target_dir does not exist" + return 1 + fi + + log_info "Processing Terraform modules in $target_dir" + + # Change to target directory + cd "$target_dir" + + # Initialize hash file + init_hashes_file "$HASHES_FILE" + + # Run terraform init only if needed + ensure_terraform_init || return 1 + + # Process modules only if modules.json exists + if [ -f "$MODULES_METADATA" ]; then + # Process registry modules + while read -r module_key; do + if [ -n "$module_key" ]; then + local module_path="$MODULES_DIR/$module_key" + if [ -d "$module_path" ]; then + if ! process_module "$module_path"; then + changes_found=1 + fi else - echo "The module $module_name has changed and its hash has been updated." + log_warning "Module path $module_path not found" fi - else - echo "Module path $module_path not found." >&2 fi - done + done < <(jq -r --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA" 2>/dev/null || echo "") + fi + + cd "$base_dir" + return $changes_found +} + +main() { + local base_dir="$PWD" + local exit_code=0 + local dirs_to_process=() + + # Check for required commands + for cmd in jq terraform tar sha256sum; do + if ! command -v "$cmd" >/dev/null 2>&1; then + log_error "Required command not found: $cmd" + exit 1 + fi + done + + # If specific directories are provided, use them + if [ "$#" -gt 0 ]; then + log_info "Processing specified directories" + dirs_to_process=("$@") else - echo "Directory $TARGET_DIR does not exist. Skipping." + log_info "No directories specified, detecting Terraform directories" + while IFS= read -r dir; do + dirs_to_process+=("$dir") + done < <(find_terraform_dirs "$base_dir") fi - cd $BASE_DIR -done \ No newline at end of file + + if [ ${#dirs_to_process[@]} -eq 0 ]; then + log_warning "No Terraform directories found" + exit 0 + fi + + log_info "Found ${#dirs_to_process[@]} Terraform directories to process" + + for target_dir in "${dirs_to_process[@]}"; do + log_info "Processing directory: $target_dir" + if ! process_directory "$target_dir" "$base_dir"; then + exit_code=1 + fi + done + + if [ $exit_code -eq 1 ]; then + log_warning "Changes detected in one or more modules" + fi + + exit $exit_code +} + +main "$@" From 5e7a8e20d66041df8c6728250236ac47f48d4382 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 19:54:10 +0100 Subject: [PATCH 03/36] feat: deeply refactored lock-modules script and pre-commit --- .pre-commit-config.yaml | 9 +- .pre-commit-hooks.yaml | 9 +- infra/scripts/lock-modules.sh | 342 ++++++++++++++++++++++------------ 3 files changed, 234 insertions(+), 126 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a6ab950..192416be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,17 @@ repos: - repo: local hooks: - - id: terraform-modules-lock - name: Lock Terraform Modules + - id: lock_modules + name: Lock Terraform Registry modules + description: 'Locks Terraform module versions and maintains hashes' entry: infra/scripts/lock-modules.sh language: script files: '\.tf$' pass_filenames: false + require_serial: true + types: [file] + exclude: | + ^.*/(?:\.terraform|modules|examples|tests)/.*$ - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.96.2 hooks: diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 66789b8f..1d3388cc 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,9 +1,14 @@ - id: lock_modules name: Lock Terraform Registry modules - description: Create the file that contains terraform registry modules hashes + description: 'Locks Terraform module versions and maintains hashes' entry: infra/scripts/lock-modules.sh language: script - + files: '\.tf$' + pass_filenames: false + require_serial: true + types: [file] + exclude: | + ^.*/(?:\.terraform|modules|examples|tests)/.*$ - id: terraform_providers_lock_staged name: Terraform Providers Lock (on staged .terraform.lock.hcl files) entry: infra/scripts/terraform_lock_precommit.sh diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 5beb43a2..767df998 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -1,225 +1,323 @@ #!/bin/bash -set -euo pipefail +# Enable error handling: exit on error (-e) and pipe failures (-o pipefail) +set -eo pipefail -# Constants -readonly MODULES_DIR=".terraform/modules" -readonly HASHES_FILE="tfmodules.lock.json" -readonly MODULES_METADATA=".terraform/modules/modules.json" -readonly REGISTRY_URL="registry.terraform.io" -readonly SCRIPT_NAME=$(basename "$0") +readonly MODULES_DIR=".terraform/modules" # Directory where Terraform stores downloaded modules +readonly HASHES_FILE="tfmodules.lock.json" # File to store module hashes +readonly MODULES_METADATA=".terraform/modules/modules.json" # Terraform's module metadata file +readonly REGISTRY_URL="registry.terraform.io" # Default Terraform registry URL +readonly SCRIPT_NAME=$(basename "$0") # Get the script name for logging -# Helper Functions -log_info() { - echo "[${SCRIPT_NAME}] INFO: $1" -} +# Enable debug mode if PRECOMMIT_DEBUG environment variable is set to 1 +# set -x enables printing of each command before execution +if [[ "${PRECOMMIT_DEBUG:-0}" -eq 1 ]]; then + set -x +fi -log_error() { - echo "[${SCRIPT_NAME}] ERROR: $1" >&2 -} +# Logging functions for different severity levels +# All output is sent to stderr (>&2) to keep stdout clean for piping +readonly PRE_COMMIT_VERBOSE="${PRE_COMMIT_VERBOSE:-0}" # Default to non-verbose if not set -log_warning() { - echo "[${SCRIPT_NAME}] WARNING: $1" >&2 +function debug() { + # Only print debug messages if both PRECOMMIT_DEBUG and VERBOSE are enabled + if [[ "${PRECOMMIT_DEBUG:-0}" -eq 1 && "${PRE_COMMIT_VERBOSE}" -eq 1 ]]; then + echo "DEBUG: $*" >&2 + fi } -find_terraform_dirs() { - local base_dir="$1" - # Find directories containing .tf files, excluding .terraform directories - find "$base_dir" -type f -name "*.tf" ! -path "*/.terraform/*" -exec dirname {} \; | sort -u +function error() { + # Always show errors regardless of verbose mode + echo "ERROR: $*" >&2 } -calculate_hash() { - local module_path="$1" - tar --exclude=.* -czf - "$module_path" 2>/dev/null | sha256sum | awk '{ print $1 }' +function warn() { + if [[ "${PRE_COMMIT_VERBOSE}" -eq 1 ]]; then + echo "WARN: $*" >&2 + fi } -init_hashes_file() { - local hashes_file="$1" - if [ ! -f "$hashes_file" ]; then - log_info "Creating new hashes file" - echo "{}" > "$hashes_file" +function info() { + if [[ "${PRE_COMMIT_VERBOSE}" -eq 1 ]]; then + echo "INFO: $*" >&2 fi } -ensure_terraform_init() { - # if [ ! -f "$MODULES_METADATA" ]; then - log_warning "Running terraform init in $(pwd)" - terraform init -input=false >/dev/null - # fi +# Find all Terraform files (.tf) in the given path +# Excludes files in .terraform/, examples/, tests/, and modules/ directories +function get_terraform_files() { + local -r path="$1" + + find "$path" \ + -type f \ + -name '*.tf' \ + -not -path "**/.terraform/*" \ + -not -path "**/examples/*" \ + -not -path "**/tests/*" \ + -not -path "**/modules/*" } -process_module() { - local module_path="$1" - local module_name=$(basename "$module_path") - local new_hash=$(calculate_hash "$module_path") - local previous_hash - - # Retrieve the previous hash - previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "$HASHES_FILE") +# Get unique directories containing Terraform files +# This helps process each Terraform configuration only once +function get_terraform_dirs() { + local -r path="${1:-.}" # Use current directory if no path provided + local -a dirs=() # Array to store unique directories + local dir - # Save the new hash - jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ - "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" - - if [ "$previous_hash" = "none" ]; then - log_info "Module $module_name: Initial hash created" - return 1 - elif [ "$previous_hash" != "$new_hash" ]; then - log_info "Module $module_name: Changes detected, updating hash" - return 1 - else - log_info "Module $module_name: No changes detected" - return 0 - fi + # Process each Terraform file and extract its directory + while IFS= read -r file; do + dir=$(dirname "$file") + # Add directory to array if not already present + if [[ ! " ${dirs[*]} " =~ " ${dir} " ]]; then + dirs+=("$dir") + fi + done < <(get_terraform_files "$path") + + # Print unique directories, handling empty array case + printf '%s\n' "${dirs[@]+"${dirs[@]}"}" } -get_modules_from_tf_files() { - # Extract module sources from .tf files - grep -h 'source[[:space:]]*=' *.tf 2>/dev/null | \ +# Extract module sources from Terraform files in current directory +function get_modules_from_tf_files() { + # Step 1: Find lines containing 'source =' in all .tf files + grep -h 'source[[:space:]]*=' ./*.tf 2>/dev/null | \ + # Step 2: Extract just the source value using sed sed -E 's/.*source[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/' | \ + # Step 3: Sort and remove duplicates sort -u || echo "" } -get_modules_from_metadata() { - # Extract module sources from modules.json - if [ -f "$MODULES_METADATA" ]; then +# Extract module sources from Terraform's modules.json metadata +function get_modules_from_metadata() { + if [[ -f "$MODULES_METADATA" ]]; then jq -r '.Modules[].Source' "$MODULES_METADATA" 2>/dev/null | sort -u || echo "" else echo "" fi } -needs_terraform_init() { +# Check if 'terraform init' needs to be run +# Returns 0 (true) if init is needed, 1 (false) if not +function needs_terraform_init() { local current_modules local metadata_modules - # If modules.json doesn't exist, we need to init - if [ ! -f "$MODULES_METADATA" ]; then - log_info "No modules.json found, terraform init needed" + # Always need init if modules.json doesn't exist + if [[ ! -f "$MODULES_METADATA" ]]; then + debug "No modules.json found, terraform init needed" return 0 fi - # Get current modules from .tf files + # Get current module sources from .tf files current_modules=$(get_modules_from_tf_files) - if [ -z "$current_modules" ]; then - log_info "No modules found in .tf files" + if [[ -z "$current_modules" ]]; then + debug "No modules found in .tf files" return 1 fi - # Get modules from metadata + # Get cached module sources from modules.json metadata_modules=$(get_modules_from_metadata) - # Compare the sorted unique lists - if [ "$current_modules" != "$metadata_modules" ]; then - log_info "Module changes detected, terraform init needed" - log_info "Current modules:" - echo "$current_modules" - log_info "Cached modules:" - echo "$metadata_modules" + # Compare current and cached modules + if [[ "$current_modules" != "$metadata_modules" ]]; then + debug "Module changes detected" + debug "Current modules: $current_modules" + debug "Cached modules: $metadata_modules" return 0 fi - log_info "No module changes detected, skipping terraform init" + debug "No module changes detected" return 1 } -ensure_terraform_init() { +# Ensure Terraform modules are initialized +function ensure_terraform_init() { if needs_terraform_init; then - log_warning "Running terraform init in $(pwd)" - if ! terraform init -input=false >/dev/null; then - log_error "Terraform init failed" + warn "Running terraform init in $(pwd)" + if ! terraform init -backend=false -input=false >/dev/null; then + error "Terraform init failed" return 1 fi fi return 0 } -process_directory() { - local target_dir="$1" - local base_dir="$2" - local changes_found=0 +# Calculate hash for a module's contents +function calculate_hash() { + local -r module_path="$1" + # Create tar archive excluding hidden files, then calculate SHA256 hash + tar --exclude=.* -cvf - "$module_path" | sha256sum | awk '{ print $1 }' +} + +# Initialize or create the hashes file if it doesn't exist +function init_hashes_file() { + local -r hashes_file="$1" + if [[ ! -f "$hashes_file" ]]; then + info "Creating new hashes file" + echo "{}" > "$hashes_file" # Create empty JSON object + git add "$hashes_file" 2>/dev/null || true # Track file in git + fi +} + +# Process a single module: calculate its hash and update the hashes file +function process_module() { + local -r module_path="$1" + local -r module_name=$(basename "$module_path") + local -r new_hash=$(calculate_hash "$module_path") + local previous_hash + + init_hashes_file "$HASHES_FILE" + + # Get previous hash from hashes file + previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") - if [ ! -d "$target_dir" ]; then - log_error "Directory $target_dir does not exist" + # Update hash in hashes file + jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ + "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + + # Handle hash changes + if [[ "$previous_hash" == "none" ]]; then + info "Module $module_name: Initial hash created" + git add "$HASHES_FILE" 2>/dev/null || true return 1 + elif [[ "$previous_hash" != "$new_hash" ]]; then + info "Module $module_name: Changes detected, updating hash" + git add "$HASHES_FILE" 2>/dev/null || true + return 1 + else + debug "Module $module_name: No changes detected" + return 0 fi +} - log_info "Processing Terraform modules in $target_dir" +function has_registry_modules() { + local modules - # Change to target directory - cd "$target_dir" + # Get all module sources + modules=$(get_modules_from_metadata) + + # Check if any module contains the registry URL + if echo "$modules" | grep -q "^$REGISTRY_URL"; then + debug "Found registry modules" + return 0 + fi + + debug "No registry modules found" + return 1 +} - # Initialize hash file - init_hashes_file "$HASHES_FILE" +# Process a single directory containing Terraform configurations +function process_directory() { + local -r target_dir="$1" # Directory to process + local -r base_dir="$2" # Original working directory + local changes_found=0 # Track if any modules changed + # Validate directory exists + if [[ ! -d "$target_dir" ]]; then + error "Directory $target_dir does not exist" + return 1 + fi + + debug "Processing Terraform modules in $target_dir" + + # Change to target directory for processing + cd "$target_dir" - # Run terraform init only if needed ensure_terraform_init || return 1 - # Process modules only if modules.json exists - if [ -f "$MODULES_METADATA" ]; then - # Process registry modules - while read -r module_key; do - if [ -n "$module_key" ]; then + # Check if lock file exists but no registry modules are present + if [[ -f "$HASHES_FILE" ]] && ! has_registry_modules; then + info "No registry modules found but lock file exists, removing it" + rm -f "$HASHES_FILE" + cd "$base_dir" + return 0 + fi + + # Only proceed if registry modules are found + if ! has_registry_modules; then + info "No registry modules found in $target_dir, skipping" + cd "$base_dir" + return 0 + fi + + # Process modules if metadata file exists + if [[ -f "$MODULES_METADATA" ]]; then + # Read each module key from the metadata file + while IFS= read -r module_key; do + if [[ -n "$module_key" ]]; then local module_path="$MODULES_DIR/$module_key" - if [ -d "$module_path" ]; then + # Process module if directory exists + if [[ -d "$module_path" ]]; then if ! process_module "$module_path"; then changes_found=1 fi else - log_warning "Module path $module_path not found" + warn "Module path $module_path not found" fi fi + # Use jq to extract module keys for modules from the specified registry done < <(jq -r --arg registry_url "$REGISTRY_URL" \ '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ "$MODULES_METADATA" 2>/dev/null || echo "") fi + # Return to original directory cd "$base_dir" return $changes_found } -main() { - local base_dir="$PWD" - local exit_code=0 - local dirs_to_process=() +# Main function - entry point of the script +function main() { + local -r base_dir="$PWD" # Store current working directory + local exit_code=0 # Track overall script success + local -a dirs_to_process=() # Array to store directories to process - # Check for required commands + # Verify all required commands are available for cmd in jq terraform tar sha256sum; do if ! command -v "$cmd" >/dev/null 2>&1; then - log_error "Required command not found: $cmd" + error "Required command not found: $cmd" exit 1 fi done - # If specific directories are provided, use them - if [ "$#" -gt 0 ]; then - log_info "Processing specified directories" - dirs_to_process=("$@") - else - log_info "No directories specified, detecting Terraform directories" - while IFS= read -r dir; do + info "Detecting Terraform directories..." + + # Build array of directories to process + # Using while read instead of mapfile for better compatibility + while IFS= read -r dir; do + if [[ -n "$dir" ]]; then dirs_to_process+=("$dir") - done < <(find_terraform_dirs "$base_dir") - fi + fi + done < <(get_terraform_dirs "$base_dir") - if [ ${#dirs_to_process[@]} -eq 0 ]; then - log_warning "No Terraform directories found" + # Exit early if no directories found + if [[ ${#dirs_to_process[@]} -eq 0 ]]; then + warn "No Terraform directories found" exit 0 fi - log_info "Found ${#dirs_to_process[@]} Terraform directories to process" + # Display list of directories to be processed + info "Found ${#dirs_to_process[@]} Terraform directories to process:" + for dir in "${dirs_to_process[@]}"; do + info " - $dir" + done + # Process each directory for target_dir in "${dirs_to_process[@]}"; do - log_info "Processing directory: $target_dir" + info "Processing directory: $target_dir" if ! process_directory "$target_dir" "$base_dir"; then exit_code=1 fi done - if [ $exit_code -eq 1 ]; then - log_warning "Changes detected in one or more modules" + # Warn if changes were detected + if [[ $exit_code -eq 1 ]]; then + warn "Changes detected in one or more modules" fi exit $exit_code } -main "$@" +# Script entry point +# Only run main if the script is being executed directly (not sourced) +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi From 1ce6b1293a85ee54dbc3743546b20ce682154ea1 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 19:55:29 +0100 Subject: [PATCH 04/36] chore: ran pre-commit --- .changeset/shaggy-vans-check.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/shaggy-vans-check.md diff --git a/.changeset/shaggy-vans-check.md b/.changeset/shaggy-vans-check.md new file mode 100644 index 00000000..df8b0b23 --- /dev/null +++ b/.changeset/shaggy-vans-check.md @@ -0,0 +1,5 @@ +--- +"pre_commit_scripts": major +--- + +First working version of the lock_modules pre-commit script From 79183c6c4484289f3f7aad7b6df49ea72f9ff4f0 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 22:22:35 +0100 Subject: [PATCH 05/36] fix: use terraform get instead of init for better performance --- infra/scripts/lock-modules.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 767df998..c8c9d25f 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -95,15 +95,15 @@ function get_modules_from_metadata() { fi } -# Check if 'terraform init' needs to be run +# Check if 'terraform get' needs to be run # Returns 0 (true) if init is needed, 1 (false) if not -function needs_terraform_init() { +function needs_terraform_get() { local current_modules local metadata_modules # Always need init if modules.json doesn't exist if [[ ! -f "$MODULES_METADATA" ]]; then - debug "No modules.json found, terraform init needed" + debug "No modules.json found, terraform get needed" return 0 fi @@ -130,11 +130,11 @@ function needs_terraform_init() { } # Ensure Terraform modules are initialized -function ensure_terraform_init() { - if needs_terraform_init; then - warn "Running terraform init in $(pwd)" - if ! terraform init -backend=false -input=false >/dev/null; then - error "Terraform init failed" +function ensure_terraform_get() { + if needs_terraform_get; then + warn "Running terraform get in $(pwd)" + if ! terraform get >/dev/null; then + error "terraform get failed" return 1 fi fi @@ -221,7 +221,7 @@ function process_directory() { # Change to target directory for processing cd "$target_dir" - ensure_terraform_init || return 1 + ensure_terraform_get || return 1 # Check if lock file exists but no registry modules are present if [[ -f "$HASHES_FILE" ]] && ! has_registry_modules; then From d5ffb94626b7051507075737efc64eef873ae284 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 22:23:08 +0100 Subject: [PATCH 06/36] fix: add -update flag to terraform get --- infra/scripts/lock-modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index c8c9d25f..98ffd384 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -133,7 +133,7 @@ function needs_terraform_get() { function ensure_terraform_get() { if needs_terraform_get; then warn "Running terraform get in $(pwd)" - if ! terraform get >/dev/null; then + if ! terraform get -update >/dev/null; then error "terraform get failed" return 1 fi From ef3367b6f4ce96387c86c09e7ae56d5b4586a98d Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 22:29:13 +0100 Subject: [PATCH 07/36] revert: fix: add -update flag to terraform get --- infra/scripts/lock-modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 98ffd384..c8c9d25f 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -133,7 +133,7 @@ function needs_terraform_get() { function ensure_terraform_get() { if needs_terraform_get; then warn "Running terraform get in $(pwd)" - if ! terraform get -update >/dev/null; then + if ! terraform get >/dev/null; then error "terraform get failed" return 1 fi From 51dceac94641ca1463d7bf61835e008f2e33a07e Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 20 Dec 2024 23:01:56 +0100 Subject: [PATCH 08/36] chore: remove unnecessary entry in pre-commit --- .pre-commit-config.yaml | 1 - .pre-commit-hooks.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 192416be..09ac96ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,6 @@ repos: files: '\.tf$' pass_filenames: false require_serial: true - types: [file] exclude: | ^.*/(?:\.terraform|modules|examples|tests)/.*$ - repo: https://github.com/antonbabenko/pre-commit-terraform diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 1d3388cc..9ca916c8 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -6,7 +6,6 @@ files: '\.tf$' pass_filenames: false require_serial: true - types: [file] exclude: | ^.*/(?:\.terraform|modules|examples|tests)/.*$ - id: terraform_providers_lock_staged From 498e30be0ec7abd03f71148b939cde1c784f8135 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Mon, 23 Dec 2024 10:59:41 +0100 Subject: [PATCH 09/36] fix: use files gotten from pre-commit args --- .pre-commit-config.yaml | 5 +-- .pre-commit-hooks.yaml | 4 +- infra/resources/dev/tfmodules.lock.json | 4 ++ infra/scripts/lock-modules.sh | 50 +++++-------------------- 4 files changed, 17 insertions(+), 46 deletions(-) create mode 100644 infra/resources/dev/tfmodules.lock.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09ac96ad..f0a5795d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,10 +7,9 @@ repos: entry: infra/scripts/lock-modules.sh language: script files: '\.tf$' - pass_filenames: false + exclude: ^.*\/modules\/.* + pass_filenames: true require_serial: true - exclude: | - ^.*/(?:\.terraform|modules|examples|tests)/.*$ - repo: https://github.com/antonbabenko/pre-commit-terraform rev: v1.96.2 hooks: diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 9ca916c8..51b25e4e 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,10 +4,8 @@ entry: infra/scripts/lock-modules.sh language: script files: '\.tf$' - pass_filenames: false + pass_filenames: true require_serial: true - exclude: | - ^.*/(?:\.terraform|modules|examples|tests)/.*$ - id: terraform_providers_lock_staged name: Terraform Providers Lock (on staged .terraform.lock.hcl files) entry: infra/scripts/terraform_lock_precommit.sh diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json new file mode 100644 index 00000000..7bec5c77 --- /dev/null +++ b/infra/resources/dev/tfmodules.lock.json @@ -0,0 +1,4 @@ +{ + "core.naming_convention": "84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652", + "core.naming_convention_gh_runner": "84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652" +} diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index c8c9d25f..0fc5e524 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -42,40 +42,6 @@ function info() { fi } -# Find all Terraform files (.tf) in the given path -# Excludes files in .terraform/, examples/, tests/, and modules/ directories -function get_terraform_files() { - local -r path="$1" - - find "$path" \ - -type f \ - -name '*.tf' \ - -not -path "**/.terraform/*" \ - -not -path "**/examples/*" \ - -not -path "**/tests/*" \ - -not -path "**/modules/*" -} - -# Get unique directories containing Terraform files -# This helps process each Terraform configuration only once -function get_terraform_dirs() { - local -r path="${1:-.}" # Use current directory if no path provided - local -a dirs=() # Array to store unique directories - local dir - - # Process each Terraform file and extract its directory - while IFS= read -r file; do - dir=$(dirname "$file") - # Add directory to array if not already present - if [[ ! " ${dirs[*]} " =~ " ${dir} " ]]; then - dirs+=("$dir") - fi - done < <(get_terraform_files "$path") - - # Print unique directories, handling empty array case - printf '%s\n' "${dirs[@]+"${dirs[@]}"}" -} - # Extract module sources from Terraform files in current directory function get_modules_from_tf_files() { # Step 1: Find lines containing 'source =' in all .tf files @@ -268,7 +234,11 @@ function process_directory() { function main() { local -r base_dir="$PWD" # Store current working directory local exit_code=0 # Track overall script success - local -a dirs_to_process=() # Array to store directories to process + local -a dirs_to_process=("$@") # Array to store directories to process + + dirs_to_process=($(for file in "${dirs_to_process[@]}"; do + dirname "$file" + done | sort -u)) # Verify all required commands are available for cmd in jq terraform tar sha256sum; do @@ -282,11 +252,11 @@ function main() { # Build array of directories to process # Using while read instead of mapfile for better compatibility - while IFS= read -r dir; do - if [[ -n "$dir" ]]; then - dirs_to_process+=("$dir") - fi - done < <(get_terraform_dirs "$base_dir") + # while IFS= read -r dir; do + # if [[ -n "$dir" ]]; then + # dirs_to_process+=("$dir") + # fi + # done < <(get_terraform_dirs "$base_dir") # Exit early if no directories found if [[ ${#dirs_to_process[@]} -eq 0 ]]; then From ff0d0c62ea8dc63214a78692eb1eedf427a1ebb2 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Mon, 23 Dec 2024 14:01:05 +0100 Subject: [PATCH 10/36] fix: tar excluding all files cause contained in hidden folder .terraform --- .github/workflows/infra_plan.yaml | 2 +- infra/resources/dev/tfmodules.lock.json | 4 ++-- infra/scripts/lock-modules.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index 8d2388c4..439678b0 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -157,7 +157,7 @@ jobs: calculate_hash() { local module_path="$1" - tar --exclude=.* -cvf - "$module_path" | sha256sum | awk '{ print $1 }' + tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' } # Iterate over modules listed in the metadata that were sourced from the Terraform registry diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json index 7bec5c77..da96394c 100644 --- a/infra/resources/dev/tfmodules.lock.json +++ b/infra/resources/dev/tfmodules.lock.json @@ -1,4 +1,4 @@ { - "core.naming_convention": "84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652", - "core.naming_convention_gh_runner": "84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652" + "dx-azure-naming-convention": "0c9ce3717cffe952bc1a3873a94374dfc9894516a08209db8879295b3068c0a2", + "dx-azure-naming-conventionz": "acc4deff5f60bec71440ab808a4721416417478d3793b2f8dc86f9ed9400230b" } diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 0fc5e524..92cca845 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -111,7 +111,7 @@ function ensure_terraform_get() { function calculate_hash() { local -r module_path="$1" # Create tar archive excluding hidden files, then calculate SHA256 hash - tar --exclude=.* -cvf - "$module_path" | sha256sum | awk '{ print $1 }' + tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' } # Initialize or create the hashes file if it doesn't exist @@ -135,7 +135,6 @@ function process_module() { # Get previous hash from hashes file previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") - # Update hash in hashes file jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" @@ -212,6 +211,7 @@ function process_directory() { local module_path="$MODULES_DIR/$module_key" # Process module if directory exists if [[ -d "$module_path" ]]; then + info "Processing module: $module_path with version $(jq -r '.version' ${module_path}/package.json)" if ! process_module "$module_path"; then changes_found=1 fi From 0a709d7ff4cc28a6fda835d8c7b370b2666139b1 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Mon, 23 Dec 2024 14:17:40 +0100 Subject: [PATCH 11/36] fix: show modules missing locks in action --- .github/workflows/infra_plan.yaml | 33 ++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index 439678b0..ee10d1a6 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -159,6 +159,9 @@ jobs: local module_path="$1" tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' } + + missing_modules=() + changed_modules=() # Iterate over modules listed in the metadata that were sourced from the Terraform registry jq -r --arg registry_url "$REGISTRY_URL" \ @@ -175,23 +178,39 @@ jobs: previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then - # The hash for the current module is not found - echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." - exit 1 + # Add to missing modules array + missing_modules+=("$module_name") else # Compare the hashes if [ "$previous_hash" == "$new_hash" ]; then echo "The module $module_name has not changed." else - echo "The module $module_name has changed!" - # Exit with an error if the module has changed - exit 1 + changed_modules+=("$module_name") fi fi else echo "Module path $module_path not found." fi - done + done < <(jq -r --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA") + + # Check if we found any issues and report them + if [ ${#missing_modules[@]} -gt 0 ] || [ ${#changed_modules[@]} -gt 0 ]; then + echo -e "\nSummary of issues found:" + + if [ ${#missing_modules[@]} -gt 0 ]; then + echo -e "\nModules missing from lock file:" + printf '%s\n' "${missing_modules[@]}" + fi + + if [ ${#changed_modules[@]} -gt 0 ]; then + echo -e "\nModules with changed hashes:" + printf '%s\n' "${changed_modules[@]}" + fi + + exit 1 + fi # Run Terraform Plan From 262b73fa418d05e3f6adb1496d5fe4e6630e4913 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Mon, 23 Dec 2024 16:05:36 +0100 Subject: [PATCH 12/36] fix: lock --- .github/workflows/infra_plan.yaml | 33 ++++++------------------- infra/resources/dev/tfmodules.lock.json | 3 +-- infra/scripts/lock-modules.sh | 8 +++--- 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index ee10d1a6..439678b0 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -159,9 +159,6 @@ jobs: local module_path="$1" tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' } - - missing_modules=() - changed_modules=() # Iterate over modules listed in the metadata that were sourced from the Terraform registry jq -r --arg registry_url "$REGISTRY_URL" \ @@ -178,39 +175,23 @@ jobs: previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then - # Add to missing modules array - missing_modules+=("$module_name") + # The hash for the current module is not found + echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." + exit 1 else # Compare the hashes if [ "$previous_hash" == "$new_hash" ]; then echo "The module $module_name has not changed." else - changed_modules+=("$module_name") + echo "The module $module_name has changed!" + # Exit with an error if the module has changed + exit 1 fi fi else echo "Module path $module_path not found." fi - done < <(jq -r --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA") - - # Check if we found any issues and report them - if [ ${#missing_modules[@]} -gt 0 ] || [ ${#changed_modules[@]} -gt 0 ]; then - echo -e "\nSummary of issues found:" - - if [ ${#missing_modules[@]} -gt 0 ]; then - echo -e "\nModules missing from lock file:" - printf '%s\n' "${missing_modules[@]}" - fi - - if [ ${#changed_modules[@]} -gt 0 ]; then - echo -e "\nModules with changed hashes:" - printf '%s\n' "${changed_modules[@]}" - fi - - exit 1 - fi + done # Run Terraform Plan diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json index da96394c..f4c7e98a 100644 --- a/infra/resources/dev/tfmodules.lock.json +++ b/infra/resources/dev/tfmodules.lock.json @@ -1,4 +1,3 @@ { - "dx-azure-naming-convention": "0c9ce3717cffe952bc1a3873a94374dfc9894516a08209db8879295b3068c0a2", - "dx-azure-naming-conventionz": "acc4deff5f60bec71440ab808a4721416417478d3793b2f8dc86f9ed9400230b" + "dx-azure-naming-conventionz": "302662f449f4359dc31a92ac3be39b3cb6311efe769c45d21149b40ab2453aa5" } diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 92cca845..656f3056 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -99,7 +99,8 @@ function needs_terraform_get() { function ensure_terraform_get() { if needs_terraform_get; then warn "Running terraform get in $(pwd)" - if ! terraform get >/dev/null; then + rm -rf "$MODULES_DIR" 2>/dev/null || true + if ! terraform get -update >/dev/null; then error "terraform get failed" return 1 fi @@ -111,7 +112,7 @@ function ensure_terraform_get() { function calculate_hash() { local -r module_path="$1" # Create tar archive excluding hidden files, then calculate SHA256 hash - tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' + tar --exclude='$module_path/.*' -cf - "$module_path" | sha256sum | awk '{ print $1 }' } # Initialize or create the hashes file if it doesn't exist @@ -188,10 +189,11 @@ function process_directory() { ensure_terraform_get || return 1 + rm -f "$HASHES_FILE" + # Check if lock file exists but no registry modules are present if [[ -f "$HASHES_FILE" ]] && ! has_registry_modules; then info "No registry modules found but lock file exists, removing it" - rm -f "$HASHES_FILE" cd "$base_dir" return 0 fi From 85dfc48eeba8cd080e1af9165dd5046dd2e5bad3 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 15:54:28 +0100 Subject: [PATCH 13/36] fix: hashing functiopn --- .github/workflows/infra_apply.yaml | 2 +- .github/workflows/infra_plan.yaml | 2 +- infra/resources/dev/tfmodules.lock.json | 3 ++- infra/scripts/lock-modules.sh | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infra_apply.yaml b/.github/workflows/infra_apply.yaml index dbb379fe..9d135235 100644 --- a/.github/workflows/infra_apply.yaml +++ b/.github/workflows/infra_apply.yaml @@ -154,7 +154,7 @@ jobs: calculate_hash() { local module_path="$1" - tar --exclude=.* -cvf - "$module_path" | sha256sum | awk '{ print $1 }' + find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' } # Iterate over modules listed in the metadata that were sourced from the Terraform registry diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index 439678b0..5df8eb90 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -157,7 +157,7 @@ jobs: calculate_hash() { local module_path="$1" - tar --exclude="$module_path/.*" -cf - "$module_path" | sha256sum | awk '{ print $1 }' + find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' } # Iterate over modules listed in the metadata that were sourced from the Terraform registry diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json index f4c7e98a..adbe9f12 100644 --- a/infra/resources/dev/tfmodules.lock.json +++ b/infra/resources/dev/tfmodules.lock.json @@ -1,3 +1,4 @@ { - "dx-azure-naming-conventionz": "302662f449f4359dc31a92ac3be39b3cb6311efe769c45d21149b40ab2453aa5" + "dx-azure-naming-convention": "807e8fafaf3cda8d1df7cc5c624715555ff150e87a8df0becc7e5cab3e54f855", + "dx-azure-naming-conventionz": "b0f4e012251e566e638cba00f25d3e9e79e7ba67058e89d6f1a6b11062e203cb" } diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 656f3056..d1811ac9 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -112,7 +112,8 @@ function ensure_terraform_get() { function calculate_hash() { local -r module_path="$1" # Create tar archive excluding hidden files, then calculate SHA256 hash - tar --exclude='$module_path/.*' -cf - "$module_path" | sha256sum | awk '{ print $1 }' + # tar --exclude='$module_path/.*' -cf - "$module_path" | sha256sum | awk '{ print $1 }' + find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' } # Initialize or create the hashes file if it doesn't exist From dfcc2b42eba7a4f24553343558c38764442d1bd3 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 15:57:40 +0100 Subject: [PATCH 14/36] fix: remove git adds in pre-commti script --- infra/scripts/lock-modules.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index d1811ac9..e2663df4 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -122,7 +122,6 @@ function init_hashes_file() { if [[ ! -f "$hashes_file" ]]; then info "Creating new hashes file" echo "{}" > "$hashes_file" # Create empty JSON object - git add "$hashes_file" 2>/dev/null || true # Track file in git fi } @@ -144,11 +143,9 @@ function process_module() { # Handle hash changes if [[ "$previous_hash" == "none" ]]; then info "Module $module_name: Initial hash created" - git add "$HASHES_FILE" 2>/dev/null || true return 1 elif [[ "$previous_hash" != "$new_hash" ]]; then info "Module $module_name: Changes detected, updating hash" - git add "$HASHES_FILE" 2>/dev/null || true return 1 else debug "Module $module_name: No changes detected" From 3f2fc5ef188b6a493f642698198520755578bed8 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 16:51:33 +0100 Subject: [PATCH 15/36] fix: try lock-modules to check --- .github/workflows/infra_apply.yaml | 72 +-------------------------- .github/workflows/infra_plan.yaml | 79 +----------------------------- 2 files changed, 4 insertions(+), 147 deletions(-) diff --git a/.github/workflows/infra_apply.yaml b/.github/workflows/infra_apply.yaml index 9d135235..8c845148 100644 --- a/.github/workflows/infra_apply.yaml +++ b/.github/workflows/infra_apply.yaml @@ -118,77 +118,9 @@ jobs: - name: Check terraform registry modules hashes id: check-terraform-registry-modules-hashes - working-directory: ${{ steps.directory.outputs.dir }} + # working-directory: ${{ steps.directory.outputs.dir }} run: | - set -euo pipefail - MODULES_DIR=".terraform/modules" - MODULES_METADATA=".terraform/modules/modules.json" - REGISTRY_URL="registry.terraform.io" - HASHES_FILE="tfmodules.lock.json" - - # Check if modules metadata exists - if [ ! -f "$MODULES_METADATA" ]; then - echo "Error: Modules metadata file $MODULES_METADATA not found. Did you run 'terraform init'?" - exit 1 - fi - - # Check for modules sourced from the Terraform registry - registry_modules=$(jq --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA") - - if [ -z "$registry_modules" ]; then - echo "No Terraform modules from the registry were found. No lock file check needed." - exit 0 - fi - - echo "Terraform modules from the registry were found. Checking for $HASHES_FILE..." - - # Check if the lock file exists if there are registry modules - if [ ! -f "$HASHES_FILE" ]; then - echo "Error: $HASHES_FILE is missing. Please ensure it exists and is up to date." - exit 1 - fi - - echo "$HASHES_FILE is present. Proceeding..." - - calculate_hash() { - local module_path="$1" - find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' - } - - # Iterate over modules listed in the metadata that were sourced from the Terraform registry - jq -r --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA" | while read -r module_key; do - - module_path="$MODULES_DIR/$module_key" - - if [ -d "$module_path" ]; then - module_name=$(basename "$module_path") - new_hash=$(calculate_hash "$module_path") - - # Retrieve the previous hash - previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") - - if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then - # The hash for the current module is not found - echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." - exit 1 - else - # Compare the hashes - if [ "$previous_hash" == "$new_hash" ]; then - echo "The module $module_name has not changed." - else - echo "The module $module_name has changed!" - # Exit with an error if the module has changed - exit 1 - fi - fi - else - echo "Module path $module_path not found." - fi - done + "$GITHUB_WORKSPACE/infra/scripts/lock-modules.sh ${{ steps.directory.outputs.dir }}" - name: Terraform Plan working-directory: ${{ steps.directory.outputs.dir }} diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index 5df8eb90..f7fc931b 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -115,84 +115,9 @@ jobs: - name: Check terraform registry modules hashes id: check-terraform-registry-modules-hashes - working-directory: ${{ steps.directory.outputs.dir }} + # working-directory: ${{ steps.directory.outputs.dir }} run: | - set -euo pipefail - MODULES_DIR=".terraform/modules" - MODULES_METADATA=".terraform/modules/modules.json" - REGISTRY_URL="registry.terraform.io" - HASHES_FILE="tfmodules.lock.json" - - # Check if modules directory exists, exit if not - if [ ! -d "$MODULES_DIR" ]; then - echo "No modules directory found. Skipping module check." - exit 0 - fi - - # Check if modules metadata exists, exit if not - if [ ! -f "$MODULES_METADATA" ]; then - echo "No modules metadata file found. Skipping module check." - exit 0 - fi - - # Check for modules sourced from the Terraform registry - registry_modules=$(jq --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA") - - if [ -z "$registry_modules" ]; then - echo "No Terraform modules from the registry were found. No lock file check needed." - exit 0 - fi - - echo "Terraform modules from the registry were found. Checking for $HASHES_FILE..." - - # Check if the lock file exists if there are registry modules - if [ ! -f "$HASHES_FILE" ]; then - echo "Error: $HASHES_FILE is missing. Please ensure it exists and is up to date." - exit 1 - fi - - echo "$HASHES_FILE is present. Proceeding..." - - calculate_hash() { - local module_path="$1" - find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' - } - - # Iterate over modules listed in the metadata that were sourced from the Terraform registry - jq -r --arg registry_url "$REGISTRY_URL" \ - '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ - "$MODULES_METADATA" | while read -r module_key; do - - module_path="$MODULES_DIR/$module_key" - - if [ -d "$module_path" ]; then - module_name=$(basename "$module_path") - new_hash=$(calculate_hash "$module_path") - - # Retrieve the previous hash - previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") - - if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then - # The hash for the current module is not found - echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." - exit 1 - else - # Compare the hashes - if [ "$previous_hash" == "$new_hash" ]; then - echo "The module $module_name has not changed." - else - echo "The module $module_name has changed!" - # Exit with an error if the module has changed - exit 1 - fi - fi - else - echo "Module path $module_path not found." - fi - done - + "$GITHUB_WORKSPACE/infra/scripts/lock-modules.sh ${{ steps.directory.outputs.dir }}" # Run Terraform Plan # The plan output is saved in a file and then processed to remove unnecessary lines From ba543daf9a4dfdd77b3ccf41d52012eb802198a6 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 16:54:41 +0100 Subject: [PATCH 16/36] fix: remove tfmodules.lock.json --- infra/resources/dev/tfmodules.lock.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 infra/resources/dev/tfmodules.lock.json diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json deleted file mode 100644 index adbe9f12..00000000 --- a/infra/resources/dev/tfmodules.lock.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "dx-azure-naming-convention": "807e8fafaf3cda8d1df7cc5c624715555ff150e87a8df0becc7e5cab3e54f855", - "dx-azure-naming-conventionz": "b0f4e012251e566e638cba00f25d3e9e79e7ba67058e89d6f1a6b11062e203cb" -} From 3818d3ba62340b70589563e2a9c5fe7928fc8583 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 16:55:52 +0100 Subject: [PATCH 17/36] test: with registry modules without lock --- infra/resources/dev/main.tf | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/infra/resources/dev/main.tf b/infra/resources/dev/main.tf index b3d3810c..a3ddf00c 100644 --- a/infra/resources/dev/main.tf +++ b/infra/resources/dev/main.tf @@ -50,4 +50,30 @@ module "core" { test_enabled = true tags = local.tags +} + +module "dx-azure-naming-convention" { + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "0.0.3" + + environment = { + prefix = "a" + env_short = "a" + location = "a" + app_name = "a" + instance_number = "a" + } +} + +module "dx-azure-naming-conventionz" { + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "0.0.1" + + environment = { + prefix = "a" + env_short = "a" + location = "a" + app_name = "a" + instance_number = "a" + } } \ No newline at end of file From 96be816a560972deba40731c50a34059c545a5ec Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 17:06:19 +0100 Subject: [PATCH 18/36] fix: jq --- infra/scripts/lock-modules.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index e2663df4..2c499e6a 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -135,10 +135,12 @@ function process_module() { init_hashes_file "$HASHES_FILE" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") + previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "${HASHES_FILE:-/dev/null}") + # Update hash in hashes file jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ - "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + # Handle hash changes if [[ "$previous_hash" == "none" ]]; then From 7d17746ba9559e11dc43d3d53ca0184107f80e04 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 17:08:51 +0100 Subject: [PATCH 19/36] test: add tfmodules --- infra/resources/dev/tfmodules.lock.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 infra/resources/dev/tfmodules.lock.json diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json new file mode 100644 index 00000000..adbe9f12 --- /dev/null +++ b/infra/resources/dev/tfmodules.lock.json @@ -0,0 +1,4 @@ +{ + "dx-azure-naming-convention": "807e8fafaf3cda8d1df7cc5c624715555ff150e87a8df0becc7e5cab3e54f855", + "dx-azure-naming-conventionz": "b0f4e012251e566e638cba00f25d3e9e79e7ba67058e89d6f1a6b11062e203cb" +} From 1d425f2b11f1da1eeb6e2589828d43cdadff050a Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Tue, 24 Dec 2024 17:30:58 +0100 Subject: [PATCH 20/36] fix --- infra/scripts/lock-modules.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 2c499e6a..205d0278 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -135,12 +135,11 @@ function process_module() { init_hashes_file "$HASHES_FILE" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "${HASHES_FILE:-/dev/null}") - + previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") + # Update hash in hashes file - jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ - "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" - + jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = "$hash"' \ + "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" # Handle hash changes if [[ "$previous_hash" == "none" ]]; then From 0fac4d4b78dda37ca0379e0bad951333f092ee3b Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:07:42 +0100 Subject: [PATCH 21/36] fix: test --- infra/scripts/lock-modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 205d0278..63364f9a 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -135,7 +135,7 @@ function process_module() { init_hashes_file "$HASHES_FILE" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") + previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "${HASHES_FILE:-/dev/null}") # Update hash in hashes file jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = "$hash"' \ From 8cead435d272cddf71dc8f1858d5b00c8375b56a Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:28:43 +0100 Subject: [PATCH 22/36] fix: test --- infra/scripts/lock-modules.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 63364f9a..c29fee42 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -135,10 +135,10 @@ function process_module() { init_hashes_file "$HASHES_FILE" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "${HASHES_FILE:-/dev/null}") + previous_hash=$(jq -r --arg module "$module_name" 'if has($module) then .[$module] else "none" end' "${HASHES_FILE:-/dev/null}") # Update hash in hashes file - jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = "$hash"' \ + jq --arg module "$module_name" --arg hash "$new_hash" '. + {($module): $hash}' \ "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" # Handle hash changes From 5e42927c8f1f793b6b30c4fa86b05c01b69468ae Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:31:34 +0100 Subject: [PATCH 23/36] fix: debug jq version --- infra/scripts/lock-modules.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index c29fee42..d2d6c731 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -135,10 +135,10 @@ function process_module() { init_hashes_file "$HASHES_FILE" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" 'if has($module) then .[$module] else "none" end' "${HASHES_FILE:-/dev/null}") - + previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") + info $(jq --version) # Update hash in hashes file - jq --arg module "$module_name" --arg hash "$new_hash" '. + {($module): $hash}' \ + jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" # Handle hash changes From 6aac4436f9dde47d678dc9ca5862a240b22ff1c0 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:32:29 +0100 Subject: [PATCH 24/36] fix: debug jq version --- infra/scripts/lock-modules.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index d2d6c731..b00a8597 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -133,10 +133,10 @@ function process_module() { local previous_hash init_hashes_file "$HASHES_FILE" + info $(jq --version) # Get previous hash from hashes file previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") - info $(jq --version) # Update hash in hashes file jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" From aab390412f2edb1ab7ff109c6498dc3cd1e85bbf Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:35:48 +0100 Subject: [PATCH 25/36] fix: test action --- .github/workflows/infra_apply.yaml | 72 ++++++++++++++++++++++- .github/workflows/infra_plan.yaml | 79 +++++++++++++++++++++++++- .github/workflows/static_analysis.yaml | 2 +- 3 files changed, 148 insertions(+), 5 deletions(-) diff --git a/.github/workflows/infra_apply.yaml b/.github/workflows/infra_apply.yaml index 8c845148..9d135235 100644 --- a/.github/workflows/infra_apply.yaml +++ b/.github/workflows/infra_apply.yaml @@ -118,9 +118,77 @@ jobs: - name: Check terraform registry modules hashes id: check-terraform-registry-modules-hashes - # working-directory: ${{ steps.directory.outputs.dir }} + working-directory: ${{ steps.directory.outputs.dir }} run: | - "$GITHUB_WORKSPACE/infra/scripts/lock-modules.sh ${{ steps.directory.outputs.dir }}" + set -euo pipefail + MODULES_DIR=".terraform/modules" + MODULES_METADATA=".terraform/modules/modules.json" + REGISTRY_URL="registry.terraform.io" + HASHES_FILE="tfmodules.lock.json" + + # Check if modules metadata exists + if [ ! -f "$MODULES_METADATA" ]; then + echo "Error: Modules metadata file $MODULES_METADATA not found. Did you run 'terraform init'?" + exit 1 + fi + + # Check for modules sourced from the Terraform registry + registry_modules=$(jq --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA") + + if [ -z "$registry_modules" ]; then + echo "No Terraform modules from the registry were found. No lock file check needed." + exit 0 + fi + + echo "Terraform modules from the registry were found. Checking for $HASHES_FILE..." + + # Check if the lock file exists if there are registry modules + if [ ! -f "$HASHES_FILE" ]; then + echo "Error: $HASHES_FILE is missing. Please ensure it exists and is up to date." + exit 1 + fi + + echo "$HASHES_FILE is present. Proceeding..." + + calculate_hash() { + local module_path="$1" + find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' + } + + # Iterate over modules listed in the metadata that were sourced from the Terraform registry + jq -r --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA" | while read -r module_key; do + + module_path="$MODULES_DIR/$module_key" + + if [ -d "$module_path" ]; then + module_name=$(basename "$module_path") + new_hash=$(calculate_hash "$module_path") + + # Retrieve the previous hash + previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") + + if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then + # The hash for the current module is not found + echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." + exit 1 + else + # Compare the hashes + if [ "$previous_hash" == "$new_hash" ]; then + echo "The module $module_name has not changed." + else + echo "The module $module_name has changed!" + # Exit with an error if the module has changed + exit 1 + fi + fi + else + echo "Module path $module_path not found." + fi + done - name: Terraform Plan working-directory: ${{ steps.directory.outputs.dir }} diff --git a/.github/workflows/infra_plan.yaml b/.github/workflows/infra_plan.yaml index f7fc931b..5df8eb90 100644 --- a/.github/workflows/infra_plan.yaml +++ b/.github/workflows/infra_plan.yaml @@ -115,9 +115,84 @@ jobs: - name: Check terraform registry modules hashes id: check-terraform-registry-modules-hashes - # working-directory: ${{ steps.directory.outputs.dir }} + working-directory: ${{ steps.directory.outputs.dir }} run: | - "$GITHUB_WORKSPACE/infra/scripts/lock-modules.sh ${{ steps.directory.outputs.dir }}" + set -euo pipefail + MODULES_DIR=".terraform/modules" + MODULES_METADATA=".terraform/modules/modules.json" + REGISTRY_URL="registry.terraform.io" + HASHES_FILE="tfmodules.lock.json" + + # Check if modules directory exists, exit if not + if [ ! -d "$MODULES_DIR" ]; then + echo "No modules directory found. Skipping module check." + exit 0 + fi + + # Check if modules metadata exists, exit if not + if [ ! -f "$MODULES_METADATA" ]; then + echo "No modules metadata file found. Skipping module check." + exit 0 + fi + + # Check for modules sourced from the Terraform registry + registry_modules=$(jq --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA") + + if [ -z "$registry_modules" ]; then + echo "No Terraform modules from the registry were found. No lock file check needed." + exit 0 + fi + + echo "Terraform modules from the registry were found. Checking for $HASHES_FILE..." + + # Check if the lock file exists if there are registry modules + if [ ! -f "$HASHES_FILE" ]; then + echo "Error: $HASHES_FILE is missing. Please ensure it exists and is up to date." + exit 1 + fi + + echo "$HASHES_FILE is present. Proceeding..." + + calculate_hash() { + local module_path="$1" + find "$module_path" -type f -not -path "$module_path/.*" | sort | xargs sha256sum | awk '{print $1}' | sha256sum | awk '{print $1}' + } + + # Iterate over modules listed in the metadata that were sourced from the Terraform registry + jq -r --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA" | while read -r module_key; do + + module_path="$MODULES_DIR/$module_key" + + if [ -d "$module_path" ]; then + module_name=$(basename "$module_path") + new_hash=$(calculate_hash "$module_path") + + # Retrieve the previous hash + previous_hash=$(jq -r --arg module "$module_name" '.[$module]' "$HASHES_FILE") + + if [ "$previous_hash" = "null" ] || [ -z "$previous_hash" ]; then + # The hash for the current module is not found + echo "Error: the hash for module $module_name is missing in $HASHES_FILE. Please ensure it exists and is up to date." + exit 1 + else + # Compare the hashes + if [ "$previous_hash" == "$new_hash" ]; then + echo "The module $module_name has not changed." + else + echo "The module $module_name has changed!" + # Exit with an error if the module has changed + exit 1 + fi + fi + else + echo "Module path $module_path not found." + fi + done + # Run Terraform Plan # The plan output is saved in a file and then processed to remove unnecessary lines diff --git a/.github/workflows/static_analysis.yaml b/.github/workflows/static_analysis.yaml index 8d592219..c8536ab7 100644 --- a/.github/workflows/static_analysis.yaml +++ b/.github/workflows/static_analysis.yaml @@ -87,6 +87,6 @@ jobs: echo "Start pre-commit on all files" - pre-commit run \ + PRE_COMMIT_VERBOSE=1 pre-commit run \ --color=always \ --all-files From 00845fbb47c6e4755d8537155e97e3028ae03e4c Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:37:43 +0100 Subject: [PATCH 26/36] fix: test action --- .github/workflows/static_analysis.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/static_analysis.yaml b/.github/workflows/static_analysis.yaml index c8536ab7..22d6ab11 100644 --- a/.github/workflows/static_analysis.yaml +++ b/.github/workflows/static_analysis.yaml @@ -73,7 +73,7 @@ jobs: echo "- FROM Base ref: origin/${{ github.event.pull_request.base.ref }}" echo "- TO Head ref: origin/${{ github.event.pull_request.head.ref }}" - pre-commit run \ + PRE_COMMIT_VERBOSE=1 pre-commit run \ --color=always \ --show-diff-on-failure \ --from-ref origin/${{ github.event.pull_request.base.ref }} \ @@ -87,6 +87,6 @@ jobs: echo "Start pre-commit on all files" - PRE_COMMIT_VERBOSE=1 pre-commit run \ + pre-commit run \ --color=always \ --all-files From 58ec4d7dd7e9ddafef4f15446b5046e78589dbba Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:40:06 +0100 Subject: [PATCH 27/36] fix: test action --- infra/scripts/lock-modules.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index b00a8597..818a00e0 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -133,7 +133,8 @@ function process_module() { local previous_hash init_hashes_file "$HASHES_FILE" - info $(jq --version) + jq_version=$(jq --version 2>&1) + info "jq version: $jq_version" # Get previous hash from hashes file previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") From 88383c0dd8c1a791813a301b66ee981958dc0689 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 11:49:43 +0100 Subject: [PATCH 28/36] fix: avoid jq keywords --- infra/scripts/lock-modules.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 818a00e0..974c17b3 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -133,13 +133,11 @@ function process_module() { local previous_hash init_hashes_file "$HASHES_FILE" - jq_version=$(jq --version 2>&1) - info "jq version: $jq_version" # Get previous hash from hashes file - previous_hash=$(jq -r --arg module "$module_name" '.[$module] // "none"' "${HASHES_FILE:-/dev/null}") + previous_hash=$(jq -r --arg module_name "$module_name" '.[$module_name] // "none"' "${HASHES_FILE:-/dev/null}") # Update hash in hashes file - jq --arg module "$module_name" --arg hash "$new_hash" '.[$module] = $hash' \ + jq --arg module_name "$module_name" --arg new_hash "$new_hash" '.[$module_name] = $new_hash' \ "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" # Handle hash changes From e1063c3cd6aa6c5374d2f08448fab193caaf66d2 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 12:33:35 +0100 Subject: [PATCH 29/36] fix: selectively delete modules hashes --- infra/scripts/lock-modules.sh | 75 ++++++++++++++--------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 974c17b3..7c5c6b62 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -61,49 +61,13 @@ function get_modules_from_metadata() { fi } -# Check if 'terraform get' needs to be run -# Returns 0 (true) if init is needed, 1 (false) if not -function needs_terraform_get() { - local current_modules - local metadata_modules - - # Always need init if modules.json doesn't exist - if [[ ! -f "$MODULES_METADATA" ]]; then - debug "No modules.json found, terraform get needed" - return 0 - fi - - # Get current module sources from .tf files - current_modules=$(get_modules_from_tf_files) - if [[ -z "$current_modules" ]]; then - debug "No modules found in .tf files" - return 1 - fi - - # Get cached module sources from modules.json - metadata_modules=$(get_modules_from_metadata) - - # Compare current and cached modules - if [[ "$current_modules" != "$metadata_modules" ]]; then - debug "Module changes detected" - debug "Current modules: $current_modules" - debug "Cached modules: $metadata_modules" - return 0 - fi - - debug "No module changes detected" - return 1 -} - # Ensure Terraform modules are initialized function ensure_terraform_get() { - if needs_terraform_get; then - warn "Running terraform get in $(pwd)" - rm -rf "$MODULES_DIR" 2>/dev/null || true - if ! terraform get -update >/dev/null; then - error "terraform get failed" - return 1 - fi + warn "Running terraform get in $(pwd)" + rm -rf "$MODULES_DIR" 2>/dev/null || true + if ! terraform get -update >/dev/null; then + error "terraform get failed" + return 1 fi return 0 } @@ -132,8 +96,6 @@ function process_module() { local -r new_hash=$(calculate_hash "$module_path") local previous_hash - init_hashes_file "$HASHES_FILE" - # Get previous hash from hashes file previous_hash=$(jq -r --arg module_name "$module_name" '.[$module_name] // "none"' "${HASHES_FILE:-/dev/null}") # Update hash in hashes file @@ -187,10 +149,11 @@ function process_directory() { ensure_terraform_get || return 1 - rm -f "$HASHES_FILE" + # Initialize hashes file if it doesn't exist + init_hashes_file "$HASHES_FILE" # Check if lock file exists but no registry modules are present - if [[ -f "$HASHES_FILE" ]] && ! has_registry_modules; then + if ! has_registry_modules; then info "No registry modules found but lock file exists, removing it" cd "$base_dir" return 0 @@ -203,9 +166,29 @@ function process_directory() { return 0 fi + init_hashes_file "$HASHES_FILE" + + # Create a temporary file to store current module keys + local temp_keys_file=$(mktemp) + # Process modules if metadata file exists if [[ -f "$MODULES_METADATA" ]]; then - # Read each module key from the metadata file + # Read each module key from the metadata file and store in temp file + jq -r --arg registry_url "$REGISTRY_URL" \ + '.Modules[] | select(.Source | contains($registry_url)) | .Key' \ + "$MODULES_METADATA" > "$temp_keys_file" 2>/dev/null + + # Remove any keys from lock file that aren't in current modules + if [[ -f "$HASHES_FILE" ]]; then + jq -r 'keys[]' "$HASHES_FILE" | while read -r existing_key; do + if ! grep -q "^${existing_key}$" "$temp_keys_file"; then + info "Removing old module key: $existing_key" + jq "del(.[\"$existing_key\"])" "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + fi + done + fi + + # Process current modules while IFS= read -r module_key; do if [[ -n "$module_key" ]]; then local module_path="$MODULES_DIR/$module_key" From 9346e65c101ad40e2e6aa5535e6f07f8f184cd12 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 12:35:36 +0100 Subject: [PATCH 30/36] chore: remove unnecessary modules used for testing --- infra/resources/dev/README.md | 2 ++ infra/resources/dev/main.tf | 26 -------------------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/infra/resources/dev/README.md b/infra/resources/dev/README.md index 03ceb90b..d1bc3961 100644 --- a/infra/resources/dev/README.md +++ b/infra/resources/dev/README.md @@ -19,6 +19,8 @@ | Name | Source | Version | |------|--------|---------| | [core](#module\_core) | ../../modules/azure_core_infra | n/a | +| [dx-azure-naming-convention](#module\_dx-azure-naming-convention) | pagopa/dx-azure-naming-convention/azurerm | 0.0.3 | +| [dx-azure-naming-conventionz](#module\_dx-azure-naming-conventionz) | pagopa/dx-azure-naming-convention/azurerm | 0.0.1 | ## Resources diff --git a/infra/resources/dev/main.tf b/infra/resources/dev/main.tf index a3ddf00c..fcfca267 100644 --- a/infra/resources/dev/main.tf +++ b/infra/resources/dev/main.tf @@ -51,29 +51,3 @@ module "core" { tags = local.tags } - -module "dx-azure-naming-convention" { - source = "pagopa/dx-azure-naming-convention/azurerm" - version = "0.0.3" - - environment = { - prefix = "a" - env_short = "a" - location = "a" - app_name = "a" - instance_number = "a" - } -} - -module "dx-azure-naming-conventionz" { - source = "pagopa/dx-azure-naming-convention/azurerm" - version = "0.0.1" - - environment = { - prefix = "a" - env_short = "a" - location = "a" - app_name = "a" - instance_number = "a" - } -} \ No newline at end of file From 812f649bbd4ad8bfc6f875c17f14d5ef69d17574 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 14:39:49 +0100 Subject: [PATCH 31/36] fix: deletion of hashes --- infra/scripts/lock-modules.sh | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index 7c5c6b62..c42687ef 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -152,13 +152,6 @@ function process_directory() { # Initialize hashes file if it doesn't exist init_hashes_file "$HASHES_FILE" - # Check if lock file exists but no registry modules are present - if ! has_registry_modules; then - info "No registry modules found but lock file exists, removing it" - cd "$base_dir" - return 0 - fi - # Only proceed if registry modules are found if ! has_registry_modules; then info "No registry modules found in $target_dir, skipping" @@ -184,6 +177,7 @@ function process_directory() { if ! grep -q "^${existing_key}$" "$temp_keys_file"; then info "Removing old module key: $existing_key" jq "del(.[\"$existing_key\"])" "$HASHES_FILE" > "tmp.$$.json" && mv "tmp.$$.json" "$HASHES_FILE" + changes_found=1 fi done fi @@ -232,14 +226,6 @@ function main() { done info "Detecting Terraform directories..." - - # Build array of directories to process - # Using while read instead of mapfile for better compatibility - # while IFS= read -r dir; do - # if [[ -n "$dir" ]]; then - # dirs_to_process+=("$dir") - # fi - # done < <(get_terraform_dirs "$base_dir") # Exit early if no directories found if [[ ${#dirs_to_process[@]} -eq 0 ]]; then From 79b16e4c495335d8bc2fca1c5c7298d09083927c Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 14:43:56 +0100 Subject: [PATCH 32/36] fix: init --- infra/scripts/lock-modules.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/infra/scripts/lock-modules.sh b/infra/scripts/lock-modules.sh index c42687ef..826c9b6e 100755 --- a/infra/scripts/lock-modules.sh +++ b/infra/scripts/lock-modules.sh @@ -151,15 +151,6 @@ function process_directory() { # Initialize hashes file if it doesn't exist init_hashes_file "$HASHES_FILE" - - # Only proceed if registry modules are found - if ! has_registry_modules; then - info "No registry modules found in $target_dir, skipping" - cd "$base_dir" - return 0 - fi - - init_hashes_file "$HASHES_FILE" # Create a temporary file to store current module keys local temp_keys_file=$(mktemp) @@ -181,6 +172,13 @@ function process_directory() { fi done fi + + # Only proceed if registry modules are found + if ! has_registry_modules; then + info "No registry modules found in $target_dir, skipping" + cd "$base_dir" + return 0 + fi # Process current modules while IFS= read -r module_key; do From 83b57f9f84da82d2f0c04c5a45f7f9395db21b45 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 2 Jan 2025 15:17:27 +0100 Subject: [PATCH 33/36] chore --- .github/workflows/static_analysis.yaml | 2 +- infra/resources/dev/.terraform.lock.hcl | 5 +++++ infra/resources/dev/README.md | 2 -- infra/resources/dev/tfmodules.lock.json | 5 +---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/static_analysis.yaml b/.github/workflows/static_analysis.yaml index 22d6ab11..8d592219 100644 --- a/.github/workflows/static_analysis.yaml +++ b/.github/workflows/static_analysis.yaml @@ -73,7 +73,7 @@ jobs: echo "- FROM Base ref: origin/${{ github.event.pull_request.base.ref }}" echo "- TO Head ref: origin/${{ github.event.pull_request.head.ref }}" - PRE_COMMIT_VERBOSE=1 pre-commit run \ + pre-commit run \ --color=always \ --show-diff-on-failure \ --from-ref origin/${{ github.event.pull_request.base.ref }} \ diff --git a/infra/resources/dev/.terraform.lock.hcl b/infra/resources/dev/.terraform.lock.hcl index ba3f6388..89bc944b 100644 --- a/infra/resources/dev/.terraform.lock.hcl +++ b/infra/resources/dev/.terraform.lock.hcl @@ -5,6 +5,7 @@ provider "registry.terraform.io/hashicorp/azuread" { version = "3.0.2" hashes = [ "h1:HNrx7UJEDY5Kbx/r1LRQDWnziqvB6x3IU+pEA8Vq7dw=", + "h1:Sbb9HgPsFPsY3Jv8Kn+eoyYXoWHLWcODr7Okh/V001k=", "h1:k0kPplqH7FWmnYeCXXrFIeCshgF1tC4LLhfk66bos3w=", "h1:sYCyzbPpSYu2XDah8XqBUITQAfB0x4j4Twh6lw2C4CA=", "h1:yQqvUtgtrYKGpIygdM8P6N+pvMWJJWIsVdPow29VE20=", @@ -30,6 +31,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { "h1:9gfR0VCUpoynii31LxsLaK9fV1blcnJQi3vnjJLSiaI=", "h1:af8gzp2nuiJVXGW2v3Ch9+W/SjbwFCTpWaylAhbiby4=", "h1:fIM8Lbg5w2m2HbETUx+aAYnTVtktETwOqnKZyVVajIo=", + "h1:leoUat4/Z1jgdSdf3d6DAPqsnAqT28bThWj5IquiXAw=", "h1:sP1K3rtDj2pVQqBBn50rOXe+QPFBAKRbI2uExOxnh3M=", "zh:016b6f4662d1cfcddbe968624e899c1a20c6df0ed5014cdeed19c3e945ea80ee", "zh:08448eeaaa9e9e84a2887282f9524faa2bb000fbdfcdac610c088a74e36e6911", @@ -51,6 +53,7 @@ provider "registry.terraform.io/hashicorp/local" { constraints = "~> 2.3" hashes = [ "h1:6NIiHWMbE9bFZaUiqC+OokdWSbW7g3+yQYnO4yvgtuY=", + "h1:6XyefmvbkprppmYbGmMcQW5NB4w6C363SSShzuhF4R0=", "h1:IyFbOIO6mhikFNL/2h1iZJ6kyN3U00jgkpCLUCThAfE=", "h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=", "h1:p99F1AoV9z51aJ4EdItxz/vLwWIyhx/0Iw7L7sWSH1o=", @@ -75,6 +78,7 @@ provider "registry.terraform.io/hashicorp/null" { "h1:+AnORRgFbRO6qqcfaQyeX80W0eX3VmjadjnUFUJTiXo=", "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", "h1:nKUqWEza6Lcv3xRlzeiRQrHtqvzX1BhIzjaOVXRYQXQ=", + "h1:obXguGZUWtNAO09f1f9Cb7hsPCOGXuGdN8bn/ohKRBQ=", "h1:zxoDtu918XPWJ/Y6s4aFrZydn6SfqkRc5Ax1ZLnC6Ew=", "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", @@ -97,6 +101,7 @@ provider "registry.terraform.io/hashicorp/random" { hashes = [ "h1:+UItZOLue/moJfnI3tqZBQbXUYR4ZnqPYfJDJPgLZy0=", "h1:Fnaec9vA8sZ8BXVlN3Xn9Jz3zghSETIKg7ch8oXhxno=", + "h1:In4XBRMdhY89yUoTUyar3wDF28RJlDpQzdjahp59FAk=", "h1:f6jXn4MCv67kgcofx9D49qx1ZEBv8oyvwKDMPBr0A24=", "h1:zG9uFP8l9u+yGZZvi5Te7PV62j50azpgwPunq2vTm1E=", "zh:04ceb65210251339f07cd4611885d242cd4d0c7306e86dda9785396807c00451", diff --git a/infra/resources/dev/README.md b/infra/resources/dev/README.md index d1bc3961..03ceb90b 100644 --- a/infra/resources/dev/README.md +++ b/infra/resources/dev/README.md @@ -19,8 +19,6 @@ | Name | Source | Version | |------|--------|---------| | [core](#module\_core) | ../../modules/azure_core_infra | n/a | -| [dx-azure-naming-convention](#module\_dx-azure-naming-convention) | pagopa/dx-azure-naming-convention/azurerm | 0.0.3 | -| [dx-azure-naming-conventionz](#module\_dx-azure-naming-conventionz) | pagopa/dx-azure-naming-convention/azurerm | 0.0.1 | ## Resources diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json index adbe9f12..0967ef42 100644 --- a/infra/resources/dev/tfmodules.lock.json +++ b/infra/resources/dev/tfmodules.lock.json @@ -1,4 +1 @@ -{ - "dx-azure-naming-convention": "807e8fafaf3cda8d1df7cc5c624715555ff150e87a8df0becc7e5cab3e54f855", - "dx-azure-naming-conventionz": "b0f4e012251e566e638cba00f25d3e9e79e7ba67058e89d6f1a6b11062e203cb" -} +{} From 207c8d0e353e6832afc595e9e61fea4362772d2d Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 3 Jan 2025 11:42:22 +0100 Subject: [PATCH 34/36] fix: remove relative referencing of modules --- infra/identity/dev/tfmodules.lock.json | 1 + infra/modules/azure_api_management/main.tf | 3 ++- infra/modules/azure_app_service/main.tf | 3 ++- infra/modules/azure_app_service_exposed/main.tf | 3 ++- infra/modules/azure_core_infra/main.tf | 3 ++- infra/modules/azure_cosmos_account/main.tf | 3 ++- infra/modules/azure_event_hub/main.tf | 3 ++- infra/modules/azure_function_app/main.tf | 3 ++- infra/modules/azure_function_app_exposed/main.tf | 3 ++- infra/modules/azure_postgres_server/main.tf | 3 ++- infra/modules/azure_storage_account/main.tf | 3 ++- .../github_selfhosted_runner_on_container_app_jobs/main.tf | 3 ++- infra/repository/tfmodules.lock.json | 1 + 13 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 infra/identity/dev/tfmodules.lock.json create mode 100644 infra/repository/tfmodules.lock.json diff --git a/infra/identity/dev/tfmodules.lock.json b/infra/identity/dev/tfmodules.lock.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/infra/identity/dev/tfmodules.lock.json @@ -0,0 +1 @@ +{} diff --git a/infra/modules/azure_api_management/main.tf b/infra/modules/azure_api_management/main.tf index 30bf2320..4d2ca551 100644 --- a/infra/modules/azure_api_management/main.tf +++ b/infra/modules/azure_api_management/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_app_service/main.tf b/infra/modules/azure_app_service/main.tf index 1b3a6262..e086c8dc 100644 --- a/infra/modules/azure_app_service/main.tf +++ b/infra/modules/azure_app_service/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_app_service_exposed/main.tf b/infra/modules/azure_app_service_exposed/main.tf index 92c65c44..9af42a45 100644 --- a/infra/modules/azure_app_service_exposed/main.tf +++ b/infra/modules/azure_app_service_exposed/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_core_infra/main.tf b/infra/modules/azure_core_infra/main.tf index 3f296988..1a6145b7 100644 --- a/infra/modules/azure_core_infra/main.tf +++ b/infra/modules/azure_core_infra/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_cosmos_account/main.tf b/infra/modules/azure_cosmos_account/main.tf index e9967c9b..0843f541 100644 --- a/infra/modules/azure_cosmos_account/main.tf +++ b/infra/modules/azure_cosmos_account/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_event_hub/main.tf b/infra/modules/azure_event_hub/main.tf index ed269424..d84b3792 100644 --- a/infra/modules/azure_event_hub/main.tf +++ b/infra/modules/azure_event_hub/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_function_app/main.tf b/infra/modules/azure_function_app/main.tf index 1b3a6262..e086c8dc 100644 --- a/infra/modules/azure_function_app/main.tf +++ b/infra/modules/azure_function_app/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_function_app_exposed/main.tf b/infra/modules/azure_function_app_exposed/main.tf index 1b3a6262..e086c8dc 100644 --- a/infra/modules/azure_function_app_exposed/main.tf +++ b/infra/modules/azure_function_app_exposed/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_postgres_server/main.tf b/infra/modules/azure_postgres_server/main.tf index 1d83fee5..6bfe296e 100644 --- a/infra/modules/azure_postgres_server/main.tf +++ b/infra/modules/azure_postgres_server/main.tf @@ -8,7 +8,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/azure_storage_account/main.tf b/infra/modules/azure_storage_account/main.tf index 385e7227..15bff1f1 100644 --- a/infra/modules/azure_storage_account/main.tf +++ b/infra/modules/azure_storage_account/main.tf @@ -12,7 +12,8 @@ provider "azurerm" { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/modules/github_selfhosted_runner_on_container_app_jobs/main.tf b/infra/modules/github_selfhosted_runner_on_container_app_jobs/main.tf index 4b14cbcc..f0289281 100644 --- a/infra/modules/github_selfhosted_runner_on_container_app_jobs/main.tf +++ b/infra/modules/github_selfhosted_runner_on_container_app_jobs/main.tf @@ -9,7 +9,8 @@ terraform { } module "naming_convention" { - source = "../azure_naming_convention" + source = "pagopa/dx-azure-naming-convention/azurerm" + version = "~> 0" environment = { prefix = var.environment.prefix diff --git a/infra/repository/tfmodules.lock.json b/infra/repository/tfmodules.lock.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/infra/repository/tfmodules.lock.json @@ -0,0 +1 @@ +{} From 248bbaaa55d69d784af798d8fcc770440032b993 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 3 Jan 2025 11:45:01 +0100 Subject: [PATCH 35/36] chore: added changeset for modules --- .changeset/tall-keys-protect.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .changeset/tall-keys-protect.md diff --git a/.changeset/tall-keys-protect.md b/.changeset/tall-keys-protect.md new file mode 100644 index 00000000..2c966a14 --- /dev/null +++ b/.changeset/tall-keys-protect.md @@ -0,0 +1,15 @@ +--- +"github_selfhosted_runner_on_container_app_jobs": patch +"azure_function_app_exposed": patch +"azure_app_service_exposed": patch +"azure_postgres_server": patch +"azure_storage_account": patch +"azure_api_management": patch +"azure_cosmos_account": patch +"azure_function_app": patch +"azure_app_service": patch +"azure_core_infra": patch +"azure_event_hub": patch +--- + +Relative module referencing substituted with terraform registry referencing From 1a3a571e3d8617b7a80c804470944540ff0b320e Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Fri, 3 Jan 2025 12:08:55 +0100 Subject: [PATCH 36/36] chore: ran pre-commit --- infra/modules/azure_api_management/README.md | 2 +- infra/modules/azure_app_service/README.md | 2 +- infra/modules/azure_app_service_exposed/README.md | 2 +- infra/modules/azure_core_infra/README.md | 2 +- infra/modules/azure_cosmos_account/README.md | 2 +- infra/modules/azure_event_hub/README.md | 2 +- infra/modules/azure_function_app/README.md | 2 +- infra/modules/azure_function_app_exposed/README.md | 2 +- infra/modules/azure_postgres_server/README.md | 2 +- infra/modules/azure_storage_account/README.md | 2 +- .../github_selfhosted_runner_on_container_app_jobs/README.md | 2 +- infra/resources/dev/tfmodules.lock.json | 4 +++- 12 files changed, 14 insertions(+), 12 deletions(-) diff --git a/infra/modules/azure_api_management/README.md b/infra/modules/azure_api_management/README.md index d92ee1b0..4bf5b0f6 100644 --- a/infra/modules/azure_api_management/README.md +++ b/infra/modules/azure_api_management/README.md @@ -48,7 +48,7 @@ module "apim" { | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_app_service/README.md b/infra/modules/azure_app_service/README.md index f7bf1b44..f5c2f1e4 100644 --- a/infra/modules/azure_app_service/README.md +++ b/infra/modules/azure_app_service/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_app_service_exposed/README.md b/infra/modules/azure_app_service_exposed/README.md index 7addaa35..88be773e 100644 --- a/infra/modules/azure_app_service_exposed/README.md +++ b/infra/modules/azure_app_service_exposed/README.md @@ -14,7 +14,7 @@ This module is used to create an Azure App Service, allowing it to be configured | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_core_infra/README.md b/infra/modules/azure_core_infra/README.md index 94d1f322..03fcba8a 100644 --- a/infra/modules/azure_core_infra/README.md +++ b/infra/modules/azure_core_infra/README.md @@ -44,7 +44,7 @@ module "core" { |------|--------|---------| | [dns](#module\_dns) | ./_modules/dns | n/a | | [key\_vault](#module\_key\_vault) | ./_modules/key_vault | n/a | -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | | [nat\_gateway](#module\_nat\_gateway) | ./_modules/nat_gateway | n/a | | [network](#module\_network) | ./_modules/networking | n/a | | [vpn](#module\_vpn) | ./_modules/vpn | n/a | diff --git a/infra/modules/azure_cosmos_account/README.md b/infra/modules/azure_cosmos_account/README.md index af31d969..5901af91 100644 --- a/infra/modules/azure_cosmos_account/README.md +++ b/infra/modules/azure_cosmos_account/README.md @@ -11,7 +11,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_event_hub/README.md b/infra/modules/azure_event_hub/README.md index cbce9a96..d4148aad 100644 --- a/infra/modules/azure_event_hub/README.md +++ b/infra/modules/azure_event_hub/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_function_app/README.md b/infra/modules/azure_function_app/README.md index d293edd0..bb5ad8ea 100644 --- a/infra/modules/azure_function_app/README.md +++ b/infra/modules/azure_function_app/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_function_app_exposed/README.md b/infra/modules/azure_function_app_exposed/README.md index d427dc4b..2175938c 100644 --- a/infra/modules/azure_function_app_exposed/README.md +++ b/infra/modules/azure_function_app_exposed/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_postgres_server/README.md b/infra/modules/azure_postgres_server/README.md index 3d8d24d2..4a65a419 100644 --- a/infra/modules/azure_postgres_server/README.md +++ b/infra/modules/azure_postgres_server/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/azure_storage_account/README.md b/infra/modules/azure_storage_account/README.md index fb7bb3b2..0d5e4d93 100644 --- a/infra/modules/azure_storage_account/README.md +++ b/infra/modules/azure_storage_account/README.md @@ -11,7 +11,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/modules/github_selfhosted_runner_on_container_app_jobs/README.md b/infra/modules/github_selfhosted_runner_on_container_app_jobs/README.md index 9eddedb1..aa928475 100644 --- a/infra/modules/github_selfhosted_runner_on_container_app_jobs/README.md +++ b/infra/modules/github_selfhosted_runner_on_container_app_jobs/README.md @@ -12,7 +12,7 @@ | Name | Source | Version | |------|--------|---------| -| [naming\_convention](#module\_naming\_convention) | ../azure_naming_convention | n/a | +| [naming\_convention](#module\_naming\_convention) | pagopa/dx-azure-naming-convention/azurerm | ~> 0 | ## Resources diff --git a/infra/resources/dev/tfmodules.lock.json b/infra/resources/dev/tfmodules.lock.json index 0967ef42..253bca02 100644 --- a/infra/resources/dev/tfmodules.lock.json +++ b/infra/resources/dev/tfmodules.lock.json @@ -1 +1,3 @@ -{} +{ + "core.naming_convention": "807e8fafaf3cda8d1df7cc5c624715555ff150e87a8df0becc7e5cab3e54f855" +}