Merge pull request #20 from anyscale/brent/vpc-proxysubnet-fix

feat: Update to TF GCP Provider v5
anyscale · Mar 27, 2024 · 832aed3 · 832aed3
2 parents 1174c44 + c06d064
commit 832aed3
Show file tree

Hide file tree

Showing 70 changed files with 461 additions and 273 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 default_stages: [commit]
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.84.0
+    rev: v1.88.2
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -12,15 +12,15 @@ repos:
       - id: terraform_tflint
         args:
           - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl
-      - id: terraform_tfsec
+      - id: terraform_trivy
       - id: terraform_checkov
         args:
           - --args=--quiet
       - id: terraform_docs
       - id: tfupdate
         name: Autoupdate Terraform Google Provider
         args:
-          - --args=provider google -v "~> 4.0"
+          - --args=provider google -v "~> 5.0"
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:

diff --git a/.tflint.hcl b/.tflint.hcl
@@ -1,6 +1,6 @@
 plugin "google" {
   enabled = true
-  version = "0.26.0"
+  version = "0.27.1"
   source  = "github.com/terraform-linters/tflint-ruleset-google"
 }
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,26 @@
-## 0.10.1 (Released)
+## 0.12.0 (Released)
+FEATURES:
+- Update to GCP Terraform Provider v5
+
+BUG FIXES:
+- VPC Submodule
+  - Proxy Subnet fix for change to GCP APIs related to IPv6
+    - GCP released a change and with v5 of the GCP Terraform Provider, the IPv6 parameter for Proxy Only Subnets is not suported/required.
+  - Subnet Names are now managed by the root module
+    - The previous behavior was causing the replacement of Subnets on any parameter change to the root module.
+- VPC Firewall Submodule
+  - Update to use VPC ID for attachment instead of dynamically identifying from the name.
+    - The previous behavior was causing the replacement of the VPC Firewall on any parameter change to the root module.
+
+BREAKING CHANGES:
+
+OTHER:
+- Change from tfsec to trivy for pre-commit
+- TFLint updated for GCP Ruleset
+- General updates/fixes for all tests in submodule examples for VPC Firewall and CloudStorage
+- pre-commit updates to the latest revisions
+
+## 0.11.0 (Released)
 FEATURES:
 - Bucket CORS Rules updates to support additional Anyscale UI functionality
 

diff --git a/README.md b/README.md
diff --git a/examples/anyscale-v2-commonname/README.md b/examples/anyscale-v2-commonname/README.md
@@ -19,7 +19,7 @@ Since this creates a new project, the user/role that is executing this example n
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 
@@ -54,4 +54,5 @@ No resources.
 | Name | Description |
 |------|-------------|
 | <a name="output_registration_command"></a> [registration\_command](#output\_registration\_command) | The Anyscale registration command. |
+| <a name="output_vpc_id"></a> [vpc\_id](#output\_vpc\_id) | The ID of the VPC. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/anyscale-v2-commonname/main.tf b/examples/anyscale-v2-commonname/main.tf
@@ -42,4 +42,7 @@ module "google_anyscale_v2_commonname" {
 
   # Cloud Storage (Bucket) Related
   anyscale_bucket_location = "US"
+
+  # Enable Cloud Logging on GCP - this will enable sending logs and metrics to GCP Logs and Monitoring
+  enable_cloud_logging_monitoring = "true"
 }
diff --git a/examples/anyscale-v2-commonname/outputs.tf b/examples/anyscale-v2-commonname/outputs.tf
@@ -17,3 +17,8 @@ output "registration_command" {
   EOT
 }
 #
+
+output "vpc_id" {
+  description = "The ID of the VPC."
+  value       = module.google_anyscale_v2_commonname.vpc_id
+}
diff --git a/examples/anyscale-v2-commonname/versions.tf b/examples/anyscale-v2-commonname/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2-existingidentityfederation/README.md b/examples/anyscale-v2-existingidentityfederation/README.md
@@ -18,7 +18,7 @@ Creates a v2 stack including:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2-existingidentityfederation/versions.tf b/examples/anyscale-v2-existingidentityfederation/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2-existingproject/README.md b/examples/anyscale-v2-existingproject/README.md
@@ -18,7 +18,7 @@ Creates a v2 stack including:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2-existingproject/versions.tf b/examples/anyscale-v2-existingproject/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2-kitchensink/README.md b/examples/anyscale-v2-kitchensink/README.md
@@ -22,7 +22,7 @@ WIP
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2-kitchensink/versions.tf b/examples/anyscale-v2-kitchensink/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2-privatenetwork/README.md b/examples/anyscale-v2-privatenetwork/README.md
@@ -23,7 +23,7 @@ via a jumpbox.
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2-privatenetwork/versions.tf b/examples/anyscale-v2-privatenetwork/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2-vpc-shared/README.md b/examples/anyscale-v2-vpc-shared/README.md
@@ -33,7 +33,7 @@ Once the Anyscale Access Service Account has been created, additionally, the ser
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2-vpc-shared/versions.tf b/examples/anyscale-v2-vpc-shared/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/examples/anyscale-v2/README.md b/examples/anyscale-v2/README.md
@@ -17,7 +17,7 @@ Creates a v2 stack including:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 

diff --git a/examples/anyscale-v2/versions.tf b/examples/anyscale-v2/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
   }
 }

diff --git a/main.tf b/main.tf
@@ -80,9 +80,15 @@ module "google_anyscale_cloudapis" {
 # VPC (Networking) Module
 # ------------------------------
 locals {
+  google_region = data.google_client_config.current.region
+
   vpc_name        = var.anyscale_vpc_name != null ? var.anyscale_vpc_name : var.anyscale_vpc_name_prefix != null ? null : local.common_name
   vpc_name_prefix = coalesce(var.anyscale_vpc_name_prefix, var.common_prefix, "anyscale-vpc-")
 
+  public_subnet_name  = coalesce(var.anyscale_vpc_public_subnet_name, try("${local.vpc_name}-${local.google_region}-${var.anyscale_vpc_public_subnet_suffix}", null), "anyscale-vpc-subnet-public")
+  private_subnet_name = coalesce(var.anyscale_vpc_private_subnet_name, try("${local.vpc_name}-${local.google_region}-${var.anyscale_vpc_private_subnet_suffix}", null), "anyscale-vpc-subnet-private")
+  proxy_subnet_name   = coalesce(var.anyscale_vpc_proxy_subnet_name, try("${local.vpc_name}-${local.google_region}-${var.anyscale_vpc_proxy_subnet_suffix}", null), "anyscale-vpc-subnet-proxy")
+
   anyscale_private_subnet_count = var.anyscale_vpc_private_subnet_cidr != null ? 1 : 0
   anyscale_proxy_subnet_count   = var.anyscale_vpc_proxy_subnet_cidr != null ? 1 : 0
   anyscale_public_subnet_count  = var.anyscale_vpc_public_subnet_cidr != null ? 1 : 0
@@ -105,9 +111,14 @@ module "google_anyscale_vpc" {
   enable_random_name_suffix = local.enable_module_random_name_suffix
   vpc_description           = var.anyscale_vpc_description
 
-  public_subnet_cidr  = var.anyscale_vpc_public_subnet_cidr
+  public_subnet_cidr = var.anyscale_vpc_public_subnet_cidr
+  public_subnet_name = local.public_subnet_name
+
   private_subnet_cidr = var.anyscale_vpc_private_subnet_cidr
-  proxy_subnet_cidr   = var.anyscale_vpc_proxy_subnet_cidr
+  private_subnet_name = local.private_subnet_name
+
+  proxy_subnet_cidr = var.anyscale_vpc_proxy_subnet_cidr
+  proxy_subnet_name = local.proxy_subnet_name
 
   create_nat = local.create_nat_gw
 }
@@ -155,6 +166,7 @@ module "google_anyscale_vpc_firewall_policy" {
   anyscale_project_id = local.vpc_project_id
 
   vpc_name = coalesce(var.existing_vpc_name, module.google_anyscale_vpc.vpc_name)
+  vpc_id   = coalesce(var.existing_vpc_id, module.google_anyscale_vpc.vpc_id)
 
   firewall_policy_name        = local.firewall_policy_name
   firewall_policy_description = var.anyscale_vpc_firewall_policy_description

diff --git a/modules/google-anyscale-cloudapis/README.md b/modules/google-anyscale-cloudapis/README.md
@@ -3,13 +3,15 @@
 [![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
 # google-anyscale-cloud-apis
 
+This sub-module enables the GCP APIs required for the Anyscale Platform. It should be used from the [root module](../../README.md).
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 
 ## Providers
 
@@ -36,8 +38,8 @@ No modules.
 | <a name="input_anyscale_activate_optional_apis"></a> [anyscale\_activate\_optional\_apis](#input\_anyscale\_activate\_optional\_apis) | (Optional) Optional APIs to activate.<br><br>A list of optional apis to activate within the project.<br><br>ex:<pre>anyscale_activate_optional_apis = [<br>  "cloudkms.googleapis.com",<br>  "containerregistry.googleapis.com",<br>  "logging.googleapis.com",<br>  "monitoring.googleapis.com",<br>  "redis.googleapis.com",<br>]</pre> | `list(string)` | `[]` | no |
 | <a name="input_anyscale_activate_required_apis"></a> [anyscale\_activate\_required\_apis](#input\_anyscale\_activate\_required\_apis) | (Optional) The list of apis to activate within the project.<br>Default enables APIs for compute, filestore, and storage. | `list(string)` | <pre>[<br>  "compute.googleapis.com",<br>  "file.googleapis.com",<br>  "storage-component.googleapis.com",<br>  "storage.googleapis.com",<br>  "certificatemanager.googleapis.com",<br>  "cloudresourcemanager.googleapis.com",<br>  "serviceusage.googleapis.com",<br>  "deploymentmanager.googleapis.com"<br>]</pre> | no |
 | <a name="input_anyscale_project_id"></a> [anyscale\_project\_id](#input\_anyscale\_project\_id) | (Optional) The ID of the project to create the resource in. If not provided, the provider project is used. Default is `null`. | `string` | `null` | no |
-| <a name="input_disable_dependent_services"></a> [disable\_dependent\_services](#input\_disable\_dependent\_services) | (Optional) Determines if services that are enabled and which depend on this service should also be disabled when this service is destroyed.<br>More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_dependent_services). | `bool` | `true` | no |
-| <a name="input_disable_services_on_destroy"></a> [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | (Optional) Determines if project services will be disabled when the resources are destroyed.<br>More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_on_destroy).<br>Default is `true`. | `bool` | `true` | no |
+| <a name="input_disable_dependent_services"></a> [disable\_dependent\_services](#input\_disable\_dependent\_services) | (Optional) Determines if services that are enabled and which depend on this service should also be disabled when this service is destroyed.<br><br>More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_dependent_services).<br><br>Setting to `true` can cause errors if disabling optional services that are required by other services.<br><br>ex:<pre>disable_dependent_services = true</pre> | `bool` | `false` | no |
+| <a name="input_disable_services_on_destroy"></a> [disable\_services\_on\_destroy](#input\_disable\_services\_on\_destroy) | (Optional) Determines if project services will be disabled when the resources are destroyed.<br><br>More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_on_destroy).<br><br>Setting to `true` can cause errors if removing optional services that are required by other services.<br><br>ex:<pre>disable_services_on_destroy = true</pre> | `bool` | `false` | no |
 | <a name="input_module_enabled"></a> [module\_enabled](#input\_module\_enabled) | (Optional) Determines whether to create the resources inside this module. Default is `true`. | `bool` | `true` | no |
 
 ## Outputs
@@ -52,5 +54,5 @@ No modules.
 [Issues]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/issues
 [badge-build]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/workflows/CI/CD%20Pipeline/badge.svg
 [badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
-[badge-tf-google]: https://img.shields.io/badge/GCP-4.+-F8991D.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/GCP-5.+-F8991D.svg?logo=terraform
 [build-status]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/actions
diff --git a/modules/google-anyscale-cloudapis/examples/README.md b/modules/google-anyscale-cloudapis/examples/README.md
@@ -6,7 +6,7 @@
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
-| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 4.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
 | <a name="requirement_google-beta"></a> [google-beta](#requirement\_google-beta) | ~> 4.0 |
 
 ## Providers

diff --git a/modules/google-anyscale-cloudapis/examples/versions.tf b/modules/google-anyscale-cloudapis/examples/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"

diff --git a/modules/google-anyscale-cloudapis/main.tf b/modules/google-anyscale-cloudapis/main.tf
@@ -42,6 +42,12 @@ resource "google_project_service" "anyscale_optional_apis" {
   service                    = each.value
   disable_on_destroy         = var.disable_services_on_destroy
   disable_dependent_services = var.disable_dependent_services
+
+  lifecycle {
+    ignore_changes = [
+      service, # Ignore changes to the service itself
+    ]
+  }
 }
 
 # **************************************************

diff --git a/modules/google-anyscale-cloudapis/variables.tf b/modules/google-anyscale-cloudapis/variables.tf
@@ -67,37 +67,36 @@ variable "anyscale_activate_optional_apis" {
   default     = []
 }
 
-# variable "activate_api_identities" {
-#   description = <<-EOT
-#     (Optional)
-#     The list of service identities (Google Managed service account for the API) to force-create for the project (e.g. in order to grant additional roles).
-#     APIs in this list will automatically be appended to `anyscale_activate_apis`.
-#     Not including the API in this list will follow the default behaviour for identity creation (which is usually when the first resource using the API is created).
-#     Any roles (e.g. service agent role) must be explicitly listed. See https://cloud.google.com/iam/docs/understanding-roles#service-agent-roles-roles for a list of related roles.
-#     Default is an empty list.
-#   EOT
-#   type = list(object({
-#     api   = string
-#     roles = list(string)
-#   }))
-#   default = []
-# }
-
 variable "disable_services_on_destroy" {
   description = <<-EOT
     (Optional) Determines if project services will be disabled when the resources are destroyed.
+
     More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_on_destroy).
-    Default is `true`.
+
+    Setting to `true` can cause errors if removing optional services that are required by other services.
+
+    ex:
+    ```
+    disable_services_on_destroy = true
+    ```
   EOT
   type        = bool
-  default     = true
+  default     = false
 }
 
 variable "disable_dependent_services" {
   description = <<-EOT
     (Optional) Determines if services that are enabled and which depend on this service should also be disabled when this service is destroyed.
+
     More information in the [terraform documentation](https://www.terraform.io/docs/providers/google/r/google_project_service.html#disable_dependent_services).
+
+    Setting to `true` can cause errors if disabling optional services that are required by other services.
+
+    ex:
+    ```
+    disable_dependent_services = true
+    ```
   EOT
-  default     = true
   type        = bool
+  default     = false
 }
diff --git a/modules/google-anyscale-cloudapis/versions.tf b/modules/google-anyscale-cloudapis/versions.tf
@@ -4,11 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 4.0"
+      version = "~> 5.0"
     }
-    # google-beta = {
-    #   source  = "hashicorp/google-beta"
-    #   version = "~> 4.0"
-    # }
   }
 }