Remove event.original removal processors (integrations 1Password to A…

…uth0) (#10417) Delete the remove event.original processors from the pipelines because as of 8.11 the Fleet final pipeline now does this automatically when the `preserve_original_event` tag is not present in the event. This bumps the minimum version to 8.11.0 because it depends on the Fleet final pipeline changes. auditd_manager was bumped to require 8.12.0 because the system tests were failing under 8.11.0. Relates #10072
elastic · Aug 13, 2024 · 989c440 · 989c440
1 parent e6362aa
commit 989c440
Show file tree

Hide file tree

Showing 45 changed files with 81 additions and 111 deletions.
diff --git a/packages/1password/changelog.yml b/packages/1password/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.30.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.29.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/1password/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/1password/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml
@@ -125,11 +125,6 @@ processors:
         - onepassword.user
         - onepassword.location # Use the included GeoIP processor
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/1password/data_stream/item_usages/elasticsearch/ingest_pipeline/default.yml b/packages/1password/data_stream/item_usages/elasticsearch/ingest_pipeline/default.yml
@@ -126,11 +126,6 @@ processors:
         - onepassword.user
         - onepassword.location # Use the included GeoIP processor
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/1password/data_stream/signin_attempts/elasticsearch/ingest_pipeline/default.yml b/packages/1password/data_stream/signin_attempts/elasticsearch/ingest_pipeline/default.yml
@@ -140,11 +140,6 @@ processors:
         - onepassword.target_user
         - onepassword.location # Use the included GeoIP processor
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/1password/manifest.yml b/packages/1password/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: 1password
 title: "1Password"
-version: "1.29.0"
+version: "1.30.0"
 description: Collect logs from 1Password with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/activemq/changelog.yml b/packages/activemq/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: 1.5.0
   changes:
     - description: Add processor support for broker, queue and topic data streams.

diff --git a/packages/activemq/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/activemq/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -62,11 +62,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
-      ignore_failure: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/activemq/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/activemq/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -77,11 +77,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/activemq/manifest.yml b/packages/activemq/manifest.yml
@@ -1,6 +1,6 @@
 name: activemq
 title: ActiveMQ
-version: "1.5.0"
+version: "1.6.0"
 description: Collect logs and metrics from ActiveMQ instances with Elastic Agent.
 type: integration
 icons:

diff --git a/packages/akamai/changelog.yml b/packages/akamai/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.25.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "2.24.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml b/packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml
@@ -442,11 +442,6 @@ processors:
         - json
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/akamai/manifest.yml b/packages/akamai/manifest.yml
@@ -1,6 +1,6 @@
 name: akamai
 title: Akamai
-version: "2.24.0"
+version: "2.25.0"
 description: Collect logs from Akamai with Elastic Agent.
 type: integration
 format_version: "3.0.2"

diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.3.0"
   changes:
     - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/amazon_security_lake/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -1337,11 +1337,6 @@ processors:
       tag: remove_duplicate_custom_fields
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/amazon_security_lake/manifest.yml b/packages/amazon_security_lake/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: amazon_security_lake
 title: Amazon Security Lake
-version: "1.3.0"
+version: "1.4.0"
 description: Collect logs from Amazon Security Lake with Elastic Agent.
 type: integration
 categories: ["aws", "security"]

diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.24.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.23.0"
   changes:
     - description: Update grok pattern to support access and error log format.

diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -231,11 +231,6 @@ processors:
         - append:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag fail-{{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
-  - remove:
-      field: event.original
-      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       source: |-

diff --git a/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/error/elasticsearch/ingest_pipeline/default.yml
@@ -136,11 +136,6 @@ processors:
           }
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.1.4
 name: apache
 title: Apache HTTP Server
-version: "1.23.0"
+version: "1.24.0"
 source:
   license: Elastic-2.0
 description: Collect logs and metrics from Apache servers with Elastic Agent.

diff --git a/packages/apache_tomcat/changelog.yml b/packages/apache_tomcat/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.7.0"
   changes:
     - description: ECS version updated to 8.11.0. Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/apache_tomcat/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache_tomcat/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -174,11 +174,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       source: |-

diff --git a/packages/apache_tomcat/data_stream/catalina/elasticsearch/ingest_pipeline/default.yml b/packages/apache_tomcat/data_stream/catalina/elasticsearch/ingest_pipeline/default.yml
@@ -65,11 +65,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       source: |-

diff --git a/packages/apache_tomcat/data_stream/localhost/elasticsearch/ingest_pipeline/default.yml b/packages/apache_tomcat/data_stream/localhost/elasticsearch/ingest_pipeline/default.yml
@@ -65,11 +65,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       source: |-

diff --git a/packages/apache_tomcat/manifest.yml b/packages/apache_tomcat/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: apache_tomcat
 title: Apache Tomcat
-version: "1.7.0"
+version: "1.8.0"
 description: Collect and parse logs and metrics from Apache Tomcat servers with Elastic Agent.
 categories: ["web", "observability"]
 type: integration

diff --git a/packages/arista_ngfw/changelog.yml b/packages/arista_ngfw/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.1.0"
   changes:
     - description: Update package spec to 3.0.3.

diff --git a/packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/arista_ngfw/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -918,11 +918,6 @@ processors:
       description: Remove Arista root field if empty
       field: arista
       if: ctx.arista != null && ctx['arista'].empty
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - remove:
       field:
         - _temp

diff --git a/packages/arista_ngfw/manifest.yml b/packages/arista_ngfw/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: arista_ngfw
 title: "Arista NG Firewall"
-version: "1.1.0"
+version: "1.2.0"
 source:
   license: "Elastic-2.0"
 description: "Collect logs and metrics from Arista NG Firewall."
@@ -10,7 +10,7 @@ categories:
   - network
 conditions:
   kibana:
-    version: "^8.10.1"
+    version: "^8.11.0"
   elastic:
     subscription: "basic"
 icons:

diff --git a/packages/atlassian_bitbucket/changelog.yml b/packages/atlassian_bitbucket/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "2.1.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_bitbucket/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -414,11 +414,6 @@ processors:
         - json
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/atlassian_bitbucket/manifest.yml b/packages/atlassian_bitbucket/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_bitbucket
 title: Atlassian Bitbucket
-version: "2.1.0"
+version: "2.2.0"
 description: Collect logs from Atlassian Bitbucket with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/atlassian_confluence/changelog.yml b/packages/atlassian_confluence/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.26.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.25.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_confluence/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -400,11 +400,6 @@ processors:
         - _tmp
         - _config
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/atlassian_confluence/manifest.yml b/packages/atlassian_confluence/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_confluence
 title: Atlassian Confluence
-version: "1.25.0"
+version: "1.26.0"
 description: Collect logs from Atlassian Confluence with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/atlassian_jira/changelog.yml b/packages/atlassian_jira/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.27.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "1.26.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/atlassian_jira/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -384,11 +384,6 @@ processors:
         - _tmp
         - _config
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/atlassian_jira/manifest.yml b/packages/atlassian_jira/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: atlassian_jira
 title: Atlassian Jira
-version: "1.26.0"
+version: "1.27.0"
 description: Collect logs from Atlassian Jira with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/auditd/changelog.yml b/packages/auditd/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.20.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10417
 - version: "3.19.2"
   changes:
     - description: Fix invalid event type.