[AWS] add support to include linked accounts when using log name pref…

…ix to select log groups (#11484) * add support to include linked accounts when using log name prefix Signed-off-by: Kavindu Dodanduwa <[email protected]> add changelog link Signed-off-by: Kavindu Dodanduwa <[email protected]> trigger build # Conflicts: # packages/aws/changelog.yml # Conflicts: # packages/aws/changelog.yml # packages/aws_logs/changelog.yml # packages/aws_logs/manifest.yml * Update packages/aws/changelog.yml Co-authored-by: kaiyan-sheng <[email protected]> * Update packages/aws/manifest.yml Co-authored-by: kaiyan-sheng <[email protected]> --------- Co-authored-by: kaiyan-sheng <[email protected]>
elastic · Nov 27, 2024 · 57a1300 · 57a1300
1 parent 6f88e20
commit 57a1300
Show file tree

Hide file tree

Showing 34 changed files with 156 additions and 5 deletions.
diff --git a/packages/aws/_dev/build/docs/README.md b/packages/aws/_dev/build/docs/README.md
@@ -64,6 +64,9 @@ You can further utilize `owning_account` parameter to refine the cross account o
 If configured, metrics will be extracted from this specified linked/owning account.
 This parameter [utilize OwningAccount](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html#API_ListMetrics_RequestParameters) parameter of the ListMetrics API request.
 
+For logs, integration supports monitoring log groups from linked accounts when log groups are extracted using `log_group_name_prefix` option.
+You can enable `include_linked_accounts_for_prefix_mode` to include log groups from linked accounts. This is disabled by default.
+
 *_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
 account and source accounts together. 
 Please see [Link monitoring accounts with source accounts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account-Setup.html#CloudWatch-Unified-Cross-Account-Setup-permissions) for more details.

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.33.0"
+  changes:
+    - description: Add option to check linked accounts when using log group prefixes to derive matching log groups
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11484
 - version: "2.32.0"
   changes:
     - description: Implemented grok processor based parsing for ipv6 & ipv4 addresses in the AWS CloudFront logs.

diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -293,6 +293,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/cloudwatch_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -14,6 +14,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/cloudwatch_logs/manifest.yml b/packages/aws/data_stream/cloudwatch_logs/manifest.yml
@@ -28,6 +28,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml
@@ -162,6 +162,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_public_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/route53_public_logs/manifest.yml b/packages/aws/data_stream/route53_public_logs/manifest.yml
@@ -27,6 +27,13 @@ streams:
         multi: false
         show_user: false
         required: false
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: log_streams
         type: text
         title: Log Streams

diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
@@ -27,6 +27,13 @@ streams:
         multi: false
         show_user: false
         required: false
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: log_streams
         type: text
         title: Log Streams

diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{/unless}}
 {{/unless}}
 

diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml
@@ -160,6 +160,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws/docs/README.md b/packages/aws/docs/README.md
@@ -64,6 +64,9 @@ You can further utilize `owning_account` parameter to refine the cross account o
 If configured, metrics will be extracted from this specified linked/owning account.
 This parameter [utilize OwningAccount](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html#API_ListMetrics_RequestParameters) parameter of the ListMetrics API request.
 
+For logs, integration supports monitoring log groups from linked accounts when log groups are extracted using `log_group_name_prefix` option.
+You can enable `include_linked_accounts_for_prefix_mode` to include log groups from linked accounts. This is disabled by default.
+
 *_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
 account and source accounts together. 
 Please see [Link monitoring accounts with source accounts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account-Setup.html#CloudWatch-Unified-Cross-Account-Setup-permissions) for more details.

diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.32.0
+version: 2.33.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/aws_bedrock/changelog.yml b/packages/aws_bedrock/changelog.yml
@@ -1,3 +1,8 @@
+- version: "0.14.0"
+  changes:
+    - description: Add option to check linked accounts when using log group prefixes to derive matching log groups
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11484
 - version: "0.13.1"
   changes:
     - description: Refactor get_guardrail_details and fix missing data.

diff --git a/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws_bedrock/data_stream/invocation/agent/stream/aws-cloudwatch.yml.hbs
@@ -11,6 +11,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{#if number_of_workers }}
 number_of_workers: {{ number_of_workers }}
 {{/if}}

diff --git a/packages/aws_bedrock/data_stream/invocation/manifest.yml b/packages/aws_bedrock/data_stream/invocation/manifest.yml
@@ -29,6 +29,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name

diff --git a/packages/aws_bedrock/manifest.yml b/packages/aws_bedrock/manifest.yml
@@ -3,12 +3,12 @@ name: aws_bedrock
 title: Amazon Bedrock
 description: Collect Amazon Bedrock model invocation logs and runtime metrics with Elastic Agent.
 type: integration
-version: "0.13.1"
+version: "0.14.0"
 categories:
   - aws
 conditions:
   kibana:
-    version: "^8.15.2"
+    version: "^8.16.0"
   elastic:
     subscription: basic
 policy_templates:

diff --git a/packages/aws_logs/changelog.yml b/packages/aws_logs/changelog.yml
@@ -1,3 +1,8 @@
+- version: "1.5.0"
+  changes:
+    - description: Add option to check linked accounts when using log group prefixes to derive matching log groups
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11484
 - version: "1.4.2"
   changes:
     - description: Add ingest pipeline input option back, which was removed in 1.4.1.

diff --git a/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs b/packages/aws_logs/data_stream/generic/agent/stream/aws-cloudwatch.yml.hbs
@@ -17,6 +17,9 @@ log_group_arn: {{ log_group_arn }}
 {{#if log_group_name_prefix }}
 log_group_name_prefix: {{ log_group_name_prefix }}
 {{/if}}
+{{#if include_linked_accounts_with_prefix }}
+include_linked_accounts_for_prefix_mode: {{ include_linked_accounts_with_prefix }}
+{{/if}}
 {{#if number_of_workers }}
 number_of_workers: {{ number_of_workers }}
 {{/if}}

diff --git a/packages/aws_logs/data_stream/generic/manifest.yml b/packages/aws_logs/data_stream/generic/manifest.yml
@@ -28,6 +28,13 @@ streams:
         required: false
         show_user: false
         description: The prefix for a group of log group names. `region_name` is required when `log_group_name_prefix` is given. `log_group_name` and `log_group_name_prefix` cannot be given at the same time.
+      - name: include_linked_accounts_with_prefix
+        type: bool
+        title: Include Linked Accounts with prefix
+        multi: false
+        required: false
+        show_user: false
+        description: Include log groups from linked accounts when using `log_group_name_prefix` to derive the monitoring log groups.
       - name: region_name
         type: text
         title: Region Name