From 91c3de77b45ff8acea2d7d5aca2ce5f68ebdb06c Mon Sep 17 00:00:00 2001 From: Brent Bain Date: Fri, 26 Jul 2024 17:39:44 -0700 Subject: [PATCH] feat: Add logging sink module Add a module for managing the _Default logging sink in Google Cloud Logging. The default configuration for Google Cloud Logging is to enable syslog for all compute instances. This can cause a high volume of logs leading to unexpected costs. This optional module allows you to disable that behavior. Additional updates include changes to the pre-commit configuration to the latest version, updates to tflint to the latest version, cleanups for the trivy linter, and updates to readme's and changelog. On branch brent/logging-sink Changes to be committed: modified: .pre-commit-config.yaml modified: .tflint.hcl modified: CHANGELOG.md modified: README.md modified: examples/anyscale-v2-commonname/main.tf modified: main.tf modified: modules/google-anyscale-cloudstorage/main.tf new file: modules/google-anyscale-loggingsink/README.md new file: modules/google-anyscale-loggingsink/data.tf new file: modules/google-anyscale-loggingsink/main.tf new file: modules/google-anyscale-loggingsink/outputs.tf new file: modules/google-anyscale-loggingsink/variables.tf new file: modules/google-anyscale-loggingsink/versions.tf modified: variables.tf --- .pre-commit-config.yaml | 4 +- .tflint.hcl | 6 +- CHANGELOG.md | 13 ++++ README.md | 6 +- examples/anyscale-v2-commonname/main.tf | 4 +- main.tf | 15 +++++ modules/google-anyscale-cloudstorage/main.tf | 1 + modules/google-anyscale-loggingsink/README.md | 54 ++++++++++++++++ modules/google-anyscale-loggingsink/data.tf | 1 + modules/google-anyscale-loggingsink/main.tf | 24 +++++++ .../google-anyscale-loggingsink/outputs.tf | 0 .../google-anyscale-loggingsink/variables.tf | 64 +++++++++++++++++++ .../google-anyscale-loggingsink/versions.tf | 14 ++++ variables.tf | 18 ++++++ 14 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 modules/google-anyscale-loggingsink/README.md create mode 100644 modules/google-anyscale-loggingsink/data.tf create mode 100644 modules/google-anyscale-loggingsink/main.tf create mode 100644 modules/google-anyscale-loggingsink/outputs.tf create mode 100644 modules/google-anyscale-loggingsink/variables.tf create mode 100644 modules/google-anyscale-loggingsink/versions.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba7ef29..49c84c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_stages: [commit] repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.2 + rev: v1.92.0 hooks: - id: terraform_fmt - id: terraform_validate @@ -22,7 +22,7 @@ repos: args: - --args=provider google -v "~> 5.0" - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/.tflint.hcl b/.tflint.hcl index 6826a13..cdd2197 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -1,12 +1,12 @@ plugin "google" { enabled = true - version = "0.27.1" + version = "0.30.0" source = "github.com/terraform-linters/tflint-ruleset-google" } config { - module = true - force = false + call_module_type = "local" + force = false } rule "terraform_required_providers" { diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb5a9e..6d3e627 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## 0.13.0 (Released) +FEATURES: +- Optional new sub-module to manage the `_Default` Logging Sink - `syslog` logs are automatically enabled for new Projects and this can lead to unexpected storage costs in Projects with many active Anyscale clusters and nodes. + +BUG FIXES: + +BREAKING CHANGES: + +OTHER: +- tflint updates +- pre-commit updates +- trivy rule update on cloudstorage module + ## 0.12.1 (Released) FEATURES: diff --git a/README.md b/README.md index c5bc0da..6dbf00a 100644 --- a/README.md +++ b/README.md @@ -94,8 +94,8 @@ None | Name | Version | |------|---------| -| [google](#provider\_google) | 4.84.0 | -| [random](#provider\_random) | 3.6.0 | +| [google](#provider\_google) | 5.38.0 | +| [random](#provider\_random) | 3.6.2 | ## Modules @@ -105,6 +105,7 @@ None | [google\_anyscale\_cloudstorage](#module\_google\_anyscale\_cloudstorage) | ./modules/google-anyscale-cloudstorage | n/a | | [google\_anyscale\_filestore](#module\_google\_anyscale\_filestore) | ./modules/google-anyscale-filestore | n/a | | [google\_anyscale\_iam](#module\_google\_anyscale\_iam) | ./modules/google-anyscale-iam | n/a | +| [google\_anyscale\_loggingsink](#module\_google\_anyscale\_loggingsink) | ./modules/google-anyscale-loggingsink | n/a | | [google\_anyscale\_memorystore](#module\_google\_anyscale\_memorystore) | ./modules/google-anyscale-memorystore | n/a | | [google\_anyscale\_project](#module\_google\_anyscale\_project) | ./modules/google-anyscale-project | n/a | | [google\_anyscale\_vpc](#module\_google\_anyscale\_vpc) | ./modules/google-anyscale-vpc | n/a | @@ -187,6 +188,7 @@ None | [enable\_anyscale\_filestore](#input\_enable\_anyscale\_filestore) | (Optional) Determines if the Anyscale Filestore is created.

ex:
enable_anyscale_filestore = true
| `bool` | `true` | no | | [enable\_anyscale\_gcs](#input\_enable\_anyscale\_gcs) | (Optional) Determines if the Anyscale Cloud Storage bucket is created.

ex:
enable_anyscale_gcs = true
| `bool` | `true` | no | | [enable\_anyscale\_iam](#input\_enable\_anyscale\_iam) | (Optional) Determines if the Anyscale IAM resources are created.

ex:
enable_anyscale_iam = true
| `bool` | `true` | no | +| [enable\_anyscale\_loggingsink](#input\_enable\_anyscale\_loggingsink) | (Optional) Determines if the Anyscale Logging Sink is executed.

This sub-module will disable sending syslog events to the `_Default` Log Sink.

ex:
enable_anyscale_loggingsink = true
| `bool` | `true` | no | | [enable\_anyscale\_memorystore](#input\_enable\_anyscale\_memorystore) | (Optional) Determines if the Anyscale Memorystore is created.

ex:
enable_anyscale_memorystore = true
| `bool` | `false` | no | | [enable\_anyscale\_vpc\_firewall](#input\_enable\_anyscale\_vpc\_firewall) | (Optional) Determines if the Anyscale VPC Firewall is created.

ex:
enable_anyscale_vpc_firewall = true
| `bool` | `true` | no | | [enable\_cloud\_logging\_monitoring](#input\_enable\_cloud\_logging\_monitoring) | (Optional) Determines if the Google Cloud Logging and Monitoring APIs are enabled.

If this is set to `true`, the following APIs will be enabled:
- logging.googleapis.com
- monitoring.googleapis.com

Additionally, the Anyscale Cluster Role will be granted access to the following roles:
- logging.logWriter
- monitoring.metricWriter
- monitoring.viewer

ex:
enable_cloud_logging_monitoring = true
| `bool` | `false` | no | diff --git a/examples/anyscale-v2-commonname/main.tf b/examples/anyscale-v2-commonname/main.tf index 3d5295e..6a3662e 100644 --- a/examples/anyscale-v2-commonname/main.tf +++ b/examples/anyscale-v2-commonname/main.tf @@ -44,5 +44,7 @@ module "google_anyscale_v2_commonname" { anyscale_bucket_location = "US" # Enable Cloud Logging on GCP - this will enable sending logs and metrics to GCP Logs and Monitoring - enable_cloud_logging_monitoring = "true" + enable_cloud_logging_monitoring = true + + enable_anyscale_loggingsink = false } diff --git a/main.tf b/main.tf index da3e8f5..7a1afe3 100644 --- a/main.tf +++ b/main.tf @@ -337,3 +337,18 @@ module "google_anyscale_memorystore" { memorystore_vpc_name = local.memorystore_vpc_name } + +# ------------------------------ +# Google LoggingSink Module +# ------------------------------ +locals { + execute_loggingsink_submodule = var.enable_anyscale_loggingsink ? true : false +} +module "google_anyscale_loggingsink" { + source = "./modules/google-anyscale-loggingsink" + module_enabled = local.execute_loggingsink_submodule + anyscale_project_id = coalesce(var.existing_project_id, module.google_anyscale_project.project_id) + depends_on = [ + module.google_anyscale_cloudapis + ] +} diff --git a/modules/google-anyscale-cloudstorage/main.tf b/modules/google-anyscale-cloudstorage/main.tf index 0369b21..389fbc8 100644 --- a/modules/google-anyscale-cloudstorage/main.tf +++ b/modules/google-anyscale-cloudstorage/main.tf @@ -115,6 +115,7 @@ locals { bucket_iam_binding_enabled = var.module_enabled && length(local.bucket_binding_roles) > 0 && length(var.bucket_iam_binding_members) > 0 ? true : false } +#trivy:ignore:avd-gcp-0007 resource "google_storage_bucket_iam_binding" "anyscale_bucket" { for_each = local.bucket_iam_binding_enabled ? local.bucket_binding_roles : [] bucket = google_storage_bucket.anyscale_bucket[0].name diff --git a/modules/google-anyscale-loggingsink/README.md b/modules/google-anyscale-loggingsink/README.md new file mode 100644 index 0000000..de66f21 --- /dev/null +++ b/modules/google-anyscale-loggingsink/README.md @@ -0,0 +1,54 @@ +[![Build Status][badge-build]][build-status] +[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases) +[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases) +# google-anyscale-loggingsink + +This sub-module modifies the `_Default` logging sink to disable syslog logging. The sub-module should be used from the [root module](../../README.md). + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [google](#requirement\_google) | ~> 5.0 | +| [random](#requirement\_random) | ~> 3.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | 5.38.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_logging_project_sink.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/logging_project_sink) | resource | +| [google_client_config.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [anyscale\_project\_id](#input\_anyscale\_project\_id) | (Optional) The ID of the project to create the resource in.

If not provided, the provider project is used.

ex:
anyscale_project_id = "my-project"
| `string` | `null` | no | +| [module\_enabled](#input\_module\_enabled) | (Optional) Determines whether to create the resources inside this module.

ex:
module_enabled = true
| `bool` | `true` | no | +| [sink\_destination](#input\_sink\_destination) | (Optional) The destination for the sink.

ex:
sink_destination = "pubsub.googleapis.com/projects/my-project/topics/my-topic"
| `string` | `null` | no | +| [sink\_exclusion\_filter](#input\_sink\_exclusion\_filter) | (Optional) The filter for the sink.

ex:
sink_filter = "resource.type=gce_instance AND logName=projects/example/logs/syslog"
| `string` | `null` | no | + +## Outputs + +No outputs. + + + +[Terraform]: https://www.terraform.io +[Issues]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/issues +[badge-build]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/workflows/CI/CD%20Pipeline/badge.svg +[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform +[badge-tf-google]: https://img.shields.io/badge/GCP-5.+-F8991D.svg?logo=terraform +[build-status]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/actions diff --git a/modules/google-anyscale-loggingsink/data.tf b/modules/google-anyscale-loggingsink/data.tf new file mode 100644 index 0000000..a988612 --- /dev/null +++ b/modules/google-anyscale-loggingsink/data.tf @@ -0,0 +1 @@ +data "google_client_config" "current" {} diff --git a/modules/google-anyscale-loggingsink/main.tf b/modules/google-anyscale-loggingsink/main.tf new file mode 100644 index 0000000..74efa48 --- /dev/null +++ b/modules/google-anyscale-loggingsink/main.tf @@ -0,0 +1,24 @@ +locals { + project_id = coalesce(var.anyscale_project_id, data.google_client_config.current.project) + + sink_destination = var.sink_destination != null ? var.sink_destination : "logging.googleapis.com/projects/${local.project_id}/locations/global/buckets/_Default" + sink_exclusion_filter = var.sink_exclusion_filter != null ? var.sink_exclusion_filter : "resource.type=gce_instance AND logName=\"projects/${local.project_id}/logs/syslog\" " +} + +# Update the _Default Logging Sink to disable syslog being sent to the default bucket +resource "google_logging_project_sink" "default" { + count = var.module_enabled ? 1 : 0 + + name = "_Default" + destination = local.sink_destination + + exclusions { + name = "anyscale" + description = "Exclude syslog files from Anyscale clusters" + filter = local.sink_exclusion_filter + } + unique_writer_identity = true + + project = var.anyscale_project_id + +} diff --git a/modules/google-anyscale-loggingsink/outputs.tf b/modules/google-anyscale-loggingsink/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/google-anyscale-loggingsink/variables.tf b/modules/google-anyscale-loggingsink/variables.tf new file mode 100644 index 0000000..72ea15a --- /dev/null +++ b/modules/google-anyscale-loggingsink/variables.tf @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------------ +# REQUIRED PARAMETERS +# These variables must be set when using this module. +# ------------------------------------------------------------------------------ + + +# ------------------------------------------------------------------------------ +# OPTIONAL PARAMETERS +# These variables have defaults, but may be overridden. +# ------------------------------------------------------------------------------ +variable "module_enabled" { + description = <<-EOF + (Optional) Determines whether to create the resources inside this module. + + ex: + ``` + module_enabled = true + ``` + EOF + type = bool + default = true +} + +variable "anyscale_project_id" { + description = <<-EOF + (Optional) The ID of the project to create the resource in. + + If not provided, the provider project is used. + + ex: + ``` + anyscale_project_id = "my-project" + ``` + EOF + type = string + default = null +} + + +variable "sink_destination" { + description = <<-EOF + (Optional) The destination for the sink. + + ex: + ``` + sink_destination = "pubsub.googleapis.com/projects/my-project/topics/my-topic" + ``` + EOF + type = string + default = null +} + +variable "sink_exclusion_filter" { + description = <<-EOF + (Optional) The filter for the sink. + + ex: + ``` + sink_filter = "resource.type=gce_instance AND logName=projects/example/logs/syslog" + ``` + EOF + type = string + default = null +} diff --git a/modules/google-anyscale-loggingsink/versions.tf b/modules/google-anyscale-loggingsink/versions.tf new file mode 100644 index 0000000..ceb1c1e --- /dev/null +++ b/modules/google-anyscale-loggingsink/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} diff --git a/variables.tf b/variables.tf index c328f6e..4b70a85 100644 --- a/variables.tf +++ b/variables.tf @@ -1537,3 +1537,21 @@ variable "anyscale_memorystore_labels" { type = map(string) default = {} } + +# -------------------------------------------- +# Anyscale LoggingSink Variables +# -------------------------------------------- +variable "enable_anyscale_loggingsink" { + description = <<-EOT + (Optional) Determines if the Anyscale Logging Sink is executed. + + This sub-module will disable sending syslog events to the `_Default` Log Sink. + + ex: + ``` + enable_anyscale_loggingsink = true + ``` + EOT + type = bool + default = true +}