ssi_all: do not remove event.original in main ingest pipeline

elastic · Dec 11, 2024 · 67a73eb · 67a73eb
1 parent 6ee9c1a
commit 67a73eb
Show file tree

Hide file tree

Showing 179 changed files with 0 additions and 894 deletions.
diff --git a/...ages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml b/...ages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml
@@ -1552,12 +1552,6 @@ processors:
     ignore_failure: true
     ignore_missing: true
 
-- remove:
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    field: event.original
-    ignore_failure: true
-    ignore_missing: true
-
 ### TODO - actually clean out the bitdefender fields properly from those we can be certain are ECS mapped
 
 - remove:

diff --git a/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml
@@ -45,11 +45,6 @@ processors:
       field:
         - json
       ignore_missing: true
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -709,11 +709,6 @@ processors:
         - bitwarden.event.ip_address
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml
@@ -103,11 +103,6 @@ processors:
         - bitwarden.group.id
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml
@@ -235,12 +235,6 @@ processors:
       tag: remove_duplicate_custom_fields
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field:
-        - event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml
@@ -305,11 +305,6 @@ processors:
         - bitwarden.policy.data.useSpecial
         - bitwarden.policy.data.useUpper
       ignore_missing: true
-  - remove:
-      field:
-        - event.original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       lang: painless
       description: Drops null/empty values recursively.

diff --git a/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml
@@ -271,11 +271,6 @@ processors:
       - _tmp
       - _conf
     ignore_missing: true
-- remove:
-    field: event.original
-    if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 - script:
     lang: painless
     description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml
@@ -830,11 +830,6 @@ processors:
             - _ingest._value.mac_address
           ignore_missing: true
           ignore_failure: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

diff --git a/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml
@@ -1477,11 +1477,6 @@ processors:
             - _ingest._value.time
           ignore_missing: true
           ignore_failure: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

diff --git a/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml
@@ -209,11 +209,6 @@ processors:
         - darktrace.system_status_alert.priority
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

diff --git a/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml
@@ -465,11 +465,6 @@ processors:
       field: json
       tag: remove_json
       ignore_missing: true
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       tag: script_to_drop_null_values
       lang: painless

diff --git a/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml
@@ -201,11 +201,6 @@ processors:
       field: json
       tag: remove_json
       ignore_missing: true
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       tag: script_to_drop_null_values
       lang: painless

diff --git a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -847,11 +847,6 @@ processors:
       field: json
       tag: remove_json
       ignore_missing: true
-  - remove:
-      field: event.original
-      tag: remove_event_original
-      ignore_missing: true
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
   - script:
       tag: script_to_drop_null_values
       lang: painless

diff --git a/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml b/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml
@@ -81,11 +81,6 @@ processors:
       value: '{{{host.name}}}'
       allow_duplicates: false
       if: ctx.host?.name != null && ctx.host?.name != ''
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml b/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml
@@ -81,11 +81,6 @@ processors:
         value: '{{{host.name}}}'
         allow_duplicates: false
         if: ctx.host?.name != null && ctx.host?.name != ''
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: event.kind

diff --git a/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml
@@ -245,12 +245,6 @@ processors:
   ####################
   ## Cleanup Fields ##
   ####################
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
-      description: "Remove event.original unless tags indicate we shold not"
   - remove:
       field:
         - cef

diff --git a/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml
@@ -136,11 +136,6 @@ processors:
         - forgerock.transactionId
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml
@@ -77,11 +77,6 @@ processors:
         - forgerock.transactionId
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml
@@ -81,11 +81,6 @@ processors:
         - forgerock.timestamp
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml
@@ -70,11 +70,6 @@ processors:
         - forgerock.eventName
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml
@@ -73,11 +73,6 @@ processors:
         - forgerock.exception
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml
@@ -120,11 +120,6 @@ processors:
         - forgerock.timestamp
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml
@@ -73,11 +73,6 @@ processors:
         - forgerock.status
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml
@@ -83,11 +83,6 @@ processors:
         - forgerock.status
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml
@@ -77,11 +77,6 @@ processors:
         - forgerock.status
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml
@@ -55,11 +55,6 @@ processors:
       field: forgerock.payload
       target_field: forgerock.idm_core
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml
@@ -100,11 +100,6 @@ processors:
         - forgerock.status
       ignore_failure: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively
       lang: painless

diff --git a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -205,11 +205,6 @@ processors:
         - json
         - _temp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags?.contains('preserve_original_event') != true
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml
@@ -250,11 +250,6 @@ processors:
       field:
         - _temp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags?.contains('preserve_original_event') != true
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml
@@ -281,11 +281,6 @@ processors:
       field:
         - _temp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags?.contains('preserve_original_event') != true
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml
@@ -213,11 +213,6 @@ processors:
         - _temp_
         - github.issues.repository
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags?.contains('preserve_original_event') != true
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml
@@ -263,11 +263,6 @@ processors:
       field:
         - _temp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags?.contains('preserve_original_event') != true
-      ignore_failure: true
-      ignore_missing: true
   - script:
       lang: painless
       description: This script processor iterates over the whole document to remove fields with null values.

diff --git a/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml
@@ -187,12 +187,6 @@ processors:
     - pipeline_error
     allow_duplicates: false
     if: ctx.error?.message != null
-- remove:
-    field: event.original
-    tag: remove_original_event
-    if: ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))
-    ignore_failure: true
-    ignore_missing: true
 - script:
     description: Drops null/empty values recursively.
     lang: painless