From 67a73eb4bbc6516e40efad1e81c404aa4b0eff6c Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 12 Dec 2024 09:52:55 +1030 Subject: [PATCH] ssi_all: do not remove event.original in main ingest pipeline --- .../elasticsearch/ingest_pipeline/default.yml | 6 ------ .../collection/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../group/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../member/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../policy/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../events/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../detection/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../device_task/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../bigipafm/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../bigipapm/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../logs/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../am_access/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../am_activity/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../am_config/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../am_core/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../idm_access/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../idm_activity/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../idm_config/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../idm_core/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../idm_sync/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dependabot/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../issues/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../api/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../pages/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../production/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../sidekiq/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../asset/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../finding/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../source/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dhcp_lease/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dns_config/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dns_data/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alerts/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../telemetry/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../events/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../log/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event_report/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../user/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../alert/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../incident/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dlp/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../web/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit_events/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../dlp_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ttp_url_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alerts/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../events/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../system/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../events/elasticsearch/ingest_pipeline/default.yml | 7 ------- .../alerts/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../incidents/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alert/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../host/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../host_profile/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../asset/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../log/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../activity/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../agent/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alert/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../group/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../issues/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alert/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../event/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../incident/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../incident/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../discover/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../reporting/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 6 ------ .../asset/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../plugin/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../scan/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../elasticsearch/ingest_pipeline/default.yml | 4 ---- .../asset/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../plugin/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../logs/elasticsearch/ingest_pipeline/default.yml | 7 ------- .../malware/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threatfox/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../url/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../threatstream/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../feed/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../intel/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ioc/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../apt/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../botnet/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../cc/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../domains/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../files/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ip/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../url/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../indicator/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 6 ------ .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alert/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../ioc/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../indicator/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../threat/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit_logs/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../time_saved/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../device/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../event/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../group/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alert/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../detection/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../log/elasticsearch/ingest_pipeline/default.yml | 4 ---- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../issue/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../alerts/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../audit/elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../elasticsearch/ingest_pipeline/default.yml | 5 ----- .../user_status/elasticsearch/ingest_pipeline/default.yml | 5 ----- 179 files changed, 894 deletions(-) diff --git a/packages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml b/packages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml index 778138af45d..2ca451f9dfb 100644 --- a/packages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitdefender/data_stream/push_notifications/elasticsearch/ingest_pipeline/default.yml @@ -1552,12 +1552,6 @@ processors: ignore_failure: true ignore_missing: true -- remove: - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - field: event.original - ignore_failure: true - ignore_missing: true - ### TODO - actually clean out the bitdefender fields properly from those we can be certain are ECS mapped - remove: diff --git a/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml index 24a20cf354e..94c56cfbaa6 100644 --- a/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitwarden/data_stream/collection/elasticsearch/ingest_pipeline/default.yml @@ -45,11 +45,6 @@ processors: field: - json ignore_missing: true - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 706c3a2c207..f9b5e985d74 100644 --- a/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitwarden/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -709,11 +709,6 @@ processors: - bitwarden.event.ip_address ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml index 35eeb371356..e0ca0935e8b 100644 --- a/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitwarden/data_stream/group/elasticsearch/ingest_pipeline/default.yml @@ -103,11 +103,6 @@ processors: - bitwarden.group.id ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml index f2fd40da9d7..0233a796e8e 100644 --- a/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitwarden/data_stream/member/elasticsearch/ingest_pipeline/default.yml @@ -235,12 +235,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml b/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml index 6c08263338d..b86aff0bcee 100644 --- a/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml +++ b/packages/bitwarden/data_stream/policy/elasticsearch/ingest_pipeline/default.yml @@ -305,11 +305,6 @@ processors: - bitwarden.policy.data.useSpecial - bitwarden.policy.data.useUpper ignore_missing: true - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 6018c943dd3..f8aa1b88c1b 100644 --- a/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cisco_meraki/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -271,11 +271,6 @@ processors: - _tmp - _conf ignore_missing: true -- remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml index 1d0331d4466..96a9e6ffa88 100644 --- a/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/darktrace/data_stream/ai_analyst_alert/elasticsearch/ingest_pipeline/default.yml @@ -830,11 +830,6 @@ processors: - _ingest._value.mac_address ignore_missing: true ignore_failure: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml index d7c081843a8..72c2a6df4d3 100644 --- a/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/darktrace/data_stream/model_breach_alert/elasticsearch/ingest_pipeline/default.yml @@ -1477,11 +1477,6 @@ processors: - _ingest._value.time ignore_missing: true ignore_failure: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml b/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml index 651cd6e1e6d..5130f4f79aa 100644 --- a/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/darktrace/data_stream/system_status_alert/elasticsearch/ingest_pipeline/default.yml @@ -209,11 +209,6 @@ processors: - darktrace.system_status_alert.priority ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml index 0f8885a7e7d..281672e1e3e 100644 --- a/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/detection/elasticsearch/ingest_pipeline/default.yml @@ -465,11 +465,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml index 38d5b5f00b5..81335b56308 100644 --- a/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/device_task/elasticsearch/ingest_pipeline/default.yml @@ -201,11 +201,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 9fd160f813f..b0a2e308158 100644 --- a/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/eset_protect/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -847,11 +847,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml b/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml index 194eab4b716..b65a9ed3740 100644 --- a/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/f5/data_stream/bigipafm/elasticsearch/ingest_pipeline/default.yml @@ -81,11 +81,6 @@ processors: value: '{{{host.name}}}' allow_duplicates: false if: ctx.host?.name != null && ctx.host?.name != '' - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml b/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml index eea9b95c865..1ca9fb7ba68 100644 --- a/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/f5/data_stream/bigipapm/elasticsearch/ingest_pipeline/default.yml @@ -81,11 +81,6 @@ processors: value: '{{{host.name}}}' allow_duplicates: false if: ctx.host?.name != null && ctx.host?.name != '' - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml index 0f1593563c3..9631ce616b0 100644 --- a/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forcepoint_web/data_stream/logs/elasticsearch/ingest_pipeline/default.yml @@ -245,12 +245,6 @@ processors: #################### ## Cleanup Fields ## #################### - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - description: "Remove event.original unless tags indicate we shold not" - remove: field: - cef diff --git a/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml index 9f40b787bbc..33994afa12d 100644 --- a/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_access/elasticsearch/ingest_pipeline/default.yml @@ -136,11 +136,6 @@ processors: - forgerock.transactionId ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml index 6277a479d5b..92bc470caad 100644 --- a/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_activity/elasticsearch/ingest_pipeline/default.yml @@ -77,11 +77,6 @@ processors: - forgerock.transactionId ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml index 796f5c5735c..3446248786d 100644 --- a/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_authentication/elasticsearch/ingest_pipeline/default.yml @@ -81,11 +81,6 @@ processors: - forgerock.timestamp ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml index 8c1b8843b3c..7359ef63ea7 100644 --- a/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_config/elasticsearch/ingest_pipeline/default.yml @@ -70,11 +70,6 @@ processors: - forgerock.eventName ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml index 9f3d6d0c78f..a77222a71f2 100644 --- a/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/am_core/elasticsearch/ingest_pipeline/default.yml @@ -73,11 +73,6 @@ processors: - forgerock.exception ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml index 4d934db0bf7..bab1917bfc1 100644 --- a/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_access/elasticsearch/ingest_pipeline/default.yml @@ -120,11 +120,6 @@ processors: - forgerock.timestamp ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml index 3dd8aa952db..28c0e9c7599 100644 --- a/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_activity/elasticsearch/ingest_pipeline/default.yml @@ -73,11 +73,6 @@ processors: - forgerock.status ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml index 47e0a9208d7..89a6d1b194e 100644 --- a/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_authentication/elasticsearch/ingest_pipeline/default.yml @@ -83,11 +83,6 @@ processors: - forgerock.status ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml index 5322e6a304d..111eaa9c575 100644 --- a/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_config/elasticsearch/ingest_pipeline/default.yml @@ -77,11 +77,6 @@ processors: - forgerock.status ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml index ea93f67f3db..fa3ea21cf71 100644 --- a/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_core/elasticsearch/ingest_pipeline/default.yml @@ -55,11 +55,6 @@ processors: field: forgerock.payload target_field: forgerock.idm_core ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml b/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml index 3a81062e7d7..dce7ede8675 100644 --- a/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml +++ b/packages/forgerock/data_stream/idm_sync/elasticsearch/ingest_pipeline/default.yml @@ -100,11 +100,6 @@ processors: - forgerock.status ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively lang: painless diff --git a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 09fce9eca48..9dfd277095a 100644 --- a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -205,11 +205,6 @@ processors: - json - _temp ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml index ad8600bb0da..167f92c6ff0 100644 --- a/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/code_scanning/elasticsearch/ingest_pipeline/default.yml @@ -250,11 +250,6 @@ processors: field: - _temp ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml index 8309e6140cf..2422631ae79 100644 --- a/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/dependabot/elasticsearch/ingest_pipeline/default.yml @@ -281,11 +281,6 @@ processors: field: - _temp ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml index 048e09c9913..3b713cca40d 100644 --- a/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/issues/elasticsearch/ingest_pipeline/default.yml @@ -213,11 +213,6 @@ processors: - _temp_ - github.issues.repository ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml index 4876219b7c3..d92113f04f4 100644 --- a/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml +++ b/packages/github/data_stream/secret_scanning/elasticsearch/ingest_pipeline/default.yml @@ -263,11 +263,6 @@ processors: field: - _temp ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml index ef243c1a628..6115c17108f 100644 --- a/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/api/elasticsearch/ingest_pipeline/default.yml @@ -187,12 +187,6 @@ processors: - pipeline_error allow_duplicates: false if: ctx.error?.message != null -- remove: - field: event.original - tag: remove_original_event - if: ctx?.tags == null || !(ctx.tags.contains("preserve_original_event")) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml index d56c248804a..e5aefcfafe1 100644 --- a/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/pages/elasticsearch/ingest_pipeline/default.yml @@ -124,11 +124,6 @@ processors: value: '{{{user.id}}}' if: ctx.user?.id != null allow_duplicates: false - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml index 2c5f089e309..4c204f759c6 100644 --- a/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/production/elasticsearch/ingest_pipeline/default.yml @@ -247,12 +247,6 @@ processors: - database allow_duplicates: false if: ctx.url?.path == '/dashboard/groups' -- remove: - field: event.original - tag: remove_original_event - if: ctx?.tags == null || !(ctx.tags.contains("preserve_original_event")) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml index 840cac0b195..bce00e4057e 100644 --- a/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gitlab/data_stream/sidekiq/elasticsearch/ingest_pipeline/default.yml @@ -149,11 +149,6 @@ processors: - rename: field: gitlab.sidekiq.gitaly_calls target_field: gitlab.gitaly.calls - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 3ababbe1bac..372c37c6253 100644 --- a/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -144,11 +144,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 2d402ab464b..a5d9418ecba 100644 --- a/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -921,12 +921,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - tag: remove_original_event - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml index 8f02877687b..d56ac724a0a 100644 --- a/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/finding/elasticsearch/ingest_pipeline/default.yml @@ -1857,11 +1857,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml index 9780a781eba..33da4779010 100644 --- a/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml +++ b/packages/google_scc/data_stream/source/elasticsearch/ingest_pipeline/default.yml @@ -72,11 +72,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml index aee99e954f2..893920126d4 100644 --- a/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/imperva_cloud_waf/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -605,11 +605,6 @@ processors: field: cef tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml index df5e988bd22..bea10f17baf 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dhcp_lease/elasticsearch/ingest_pipeline/default.yml @@ -230,11 +230,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml index 3a241353981..dfce7ef4d3a 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_config/elasticsearch/ingest_pipeline/default.yml @@ -1982,11 +1982,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml index e54aa140b56..bbc2e5c5dba 100644 --- a/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_bloxone_ddi/data_stream/dns_data/elasticsearch/ingest_pipeline/default.yml @@ -452,11 +452,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml index bce02fa4754..127db73bba1 100644 --- a/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/infoblox_nios/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -169,11 +169,6 @@ processors: return false; } drop(ctx); - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: - _conf diff --git a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml index bba911f7f9e..fae7fdbe302 100644 --- a/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_compliance_reporter/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -22,11 +22,6 @@ processors: - pipeline: name: '{{ IngestPipeline "pipeline_event" }}' if: "['AUDIO_VIDEO_DEVICE_EVENT','AUDIT_CLASS_VERIFICATION_EVENT','COMPLIANCE_REPORTER_TAMPER_EVENT','FILE_EVENT','GATEKEEPER_INFO_EVENT','GATEKEEPER_MANUAL_OVERRIDES','GATEKEEPER_QUARANTINE_LOG','HARDWARE_EVENT','LICENSE_INFO_EVENT','PREFERENCE_LIST_EVENT','PRINT_EVENT_INFORMATION','PROHIBITED_APP_BLOCKED','SIGNAL_EVENT','UNIFIED_LOG_EVENT','XPROTECT_DEFINITIONS_VERSION_INFO','XPROTECT_EVENT_LOG'].contains(ctx.json?.header?.event_name)" - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: json ignore_missing: true diff --git a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 95aaaf7e2d9..f84c31f3f1b 100644 --- a/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -465,10 +465,6 @@ processors: - jamf_protect - message ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml index 409a6a5cce6..973cb18fc71 100644 --- a/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/telemetry/elasticsearch/ingest_pipeline/default.yml @@ -307,10 +307,6 @@ processors: - pipeline: name: '{{ IngestPipeline "pipeline_event_bios_uefi" }}' if: ctx.event.action == "bios_uefi" - - remove: - field: event.original - if: ctx.tags == null || !ctx.tags.contains('preserve_original_event') - ignore_failure: true ############# ## Cleanup ## diff --git a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml index 4cfc09fab88..5570bdcf6f2 100644 --- a/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/telemetry_legacy/elasticsearch/ingest_pipeline/default.yml @@ -24,10 +24,6 @@ processors: - pipeline: name: '{{ IngestPipeline "pipeline_event" }}' if: "['FILE_COLLECTION_EVENT','PLAINTEXT_LOG_COLLECTION'].contains(ctx.json?.header?.event_name)" - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - remove: field: json ignore_missing: true diff --git a/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml index f019931a1e7..41a9fe8aa92 100644 --- a/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/web_threat_events/elasticsearch/ingest_pipeline/default.yml @@ -220,10 +220,6 @@ processors: - jamf_protect - message ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml b/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml index 5419eea5c1b..ee81fc07a9d 100644 --- a/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jamf_protect/data_stream/web_traffic_events/elasticsearch/ingest_pipeline/default.yml @@ -243,10 +243,6 @@ processors: - jamf_protect - message ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 23777e52455..1d6db4ffec5 100644 --- a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -294,12 +294,6 @@ processors: - jumpcloud.event.process_name ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - description: Remove event.original unless instructed to retain it - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 0c42a6a789e..acce1f95538 100644 --- a/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/keycloak/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -47,12 +47,6 @@ processors: - _tmp ignore_missing: true tag: remove_tmp - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - tag: remove_event_original - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml index 665682eb884..3c9bfb18907 100644 --- a/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/detailed_shared_folder/elasticsearch/ingest_pipeline/default.yml @@ -114,11 +114,6 @@ processors: - lastpass.detailed_shared_folder.user.name ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml index bf92333ed55..18efc1b73be 100644 --- a/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/event_report/elasticsearch/ingest_pipeline/default.yml @@ -491,11 +491,6 @@ processors: - lastpass.event_report.data.group_name ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml b/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml index 264acc2a3fd..75e519c870c 100644 --- a/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lastpass/data_stream/user/elasticsearch/ingest_pipeline/default.yml @@ -214,11 +214,6 @@ processors: - lastpass.user.group ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 086017a3b6c..dfaf7a1a214 100644 --- a/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/lyve_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -24,12 +24,6 @@ processors: - log.file.path - "@timestamp" ignore_missing: true - - remove: - description: Remove event.original if specified in the integration config. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 0593be0b360..b59468058db 100644 --- a/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -2513,11 +2513,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 6f26c8f958a..7807f623ec2 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -113,11 +113,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. tag: script_drop_empty_fields diff --git a/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml index f0048427ce4..1b07e69b2ff 100644 --- a/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml +++ b/packages/m365_defender/data_stream/incident/elasticsearch/ingest_pipeline/default.yml @@ -2335,10 +2335,6 @@ processors: - _ingest._value.registry_value - _ingest._value.security_group_id ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml index b90bac63e5c..c4a11d2f48a 100644 --- a/packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/m365_defender/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -381,11 +381,6 @@ processors: field: json target_field: m365_defender ignore_missing: true - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index b0728e19285..4dc96fee4f1 100644 --- a/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mattermost/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -412,11 +412,6 @@ processors: field: - json ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml b/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml index ac726d70faa..1999e30e88f 100644 --- a/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml +++ b/packages/menlo/data_stream/dlp/elasticsearch/ingest_pipeline/default.yml @@ -163,11 +163,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml index cf6cfc2b561..8fd6ccfe571 100644 --- a/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml +++ b/packages/menlo/data_stream/web/elasticsearch/ingest_pipeline/default.yml @@ -309,11 +309,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 273ff2e9aa3..edc492c0d65 100644 --- a/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -1678,11 +1678,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 60d07fa1c4d..bf81a4baa62 100644 --- a/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_defender_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -336,11 +336,6 @@ processors: field: microsoft.defender_endpoint.investigationId type: string ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml index ad08343df6c..408e1d4f149 100644 --- a/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_exchange_online_message_trace/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -308,11 +308,6 @@ processors: field: _conf ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml index 99473406d69..7a058dc74ea 100644 --- a/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/archive_search_logs/elasticsearch/ingest_pipeline/default.yml @@ -141,11 +141,6 @@ processors: - mimecast.source - user.parts ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true # Error handling on_failure: diff --git a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml index f50e19a1fcf..76d24d84f3c 100644 --- a/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/audit_events/elasticsearch/ingest_pipeline/default.yml @@ -344,11 +344,6 @@ processors: - mimecast.rest_of_event_info ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - remove: description: Remove 'source.ip' if 'auditType' is not set. field: source.ip diff --git a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml index ee9e240c11f..df6903f7bd8 100644 --- a/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/dlp_logs/elasticsearch/ingest_pipeline/default.yml @@ -91,11 +91,6 @@ processors: field: - mimecast ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true # Error handling on_failure: diff --git a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml index ae6c47c4c59..e6355ac8ed3 100644 --- a/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/message_release_logs/elasticsearch/ingest_pipeline/default.yml @@ -220,11 +220,6 @@ processors: return false; } drop(ctx); - - remove: - field: event.original - if: ctx.tags == null || !ctx.tags.contains('preserve_original_event') - ignore_failure: true - ignore_missing: true - rename: tag: move_fields_into_place field: mimecast diff --git a/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml index 963b95d3638..a62cfb660dc 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_customer/elasticsearch/ingest_pipeline/default.yml @@ -197,11 +197,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - mimecast.created diff --git a/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml index 3a8d37b6497..e7743432263 100644 --- a/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/threat_intel_malware_grid/elasticsearch/ingest_pipeline/default.yml @@ -195,11 +195,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - mimecast.created diff --git a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml index 9f235962343..60978d0f3f2 100644 --- a/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ap_logs/elasticsearch/ingest_pipeline/default.yml @@ -134,11 +134,6 @@ processors: - mimecast.senderAddress - mimecast.recipientAddress ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true # Error handling on_failure: - set: diff --git a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml index 52bcbe4d2be..c663b0b82b5 100644 --- a/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_ip_logs/elasticsearch/ingest_pipeline/default.yml @@ -101,11 +101,6 @@ processors: - mimecast.senderAddress - mimecast.recipientAddress ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true # Error handling on_failure: - set: diff --git a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml index d0071cb8b1a..a210b158a82 100644 --- a/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mimecast/data_stream/ttp_url_logs/elasticsearch/ingest_pipeline/default.yml @@ -133,11 +133,6 @@ processors: - mimecast.fromUserEmailAddress - mimecast.userEmailAddress ignore_missing: true - - remove: - description: Remove 'event.original' if 'preserve_original_event' is not set. - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true # Error handling on_failure: - set: diff --git a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 4d735b9e1cb..3ab133d7c03 100644 --- a/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -1376,11 +1376,6 @@ processors: return false; } dropEmptyFields(ctx); - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 59a15a86940..dfe7160197a 100644 --- a/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/netskope/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -1119,11 +1119,6 @@ processors: return false; } dropEmptyFields(ctx); - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index d7c78acd4d3..a3c16274159 100644 --- a/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1276,11 +1276,6 @@ processors: field: - _conf ignore_missing: true - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml index 8f70f770080..a980d2725e7 100644 --- a/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml +++ b/packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml @@ -606,11 +606,6 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: _conf ignore_missing: true diff --git a/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml index 4ca8c228224..bb0dfa6908a 100755 --- a/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/opencanary/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -774,13 +774,6 @@ processors: return false; } dropEmptyFields(ctx); - - remove: - description: Remove event.original if it has not been requested - tag: remove_event_original - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - append: diff --git a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 1a4812f6cb0..bba89b5a418 100644 --- a/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw_cortex_xdr/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -556,11 +556,6 @@ processors: - panw_cortex.xdr.mitre_technique_id_and_name - panw_cortex.xdr.mitre_tactic_id_and_name ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml b/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml index 7ec259491a7..c12b86b5679 100644 --- a/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml +++ b/packages/panw_cortex_xdr/data_stream/incidents/elasticsearch/ingest_pipeline/default.yml @@ -208,11 +208,6 @@ processors: - panw_cortex.xdr.mitre_techniques_id_and_names - panw_cortex.xdr.mitre_tactics_id_and_names ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index f74a9594541..0b59702d48d 100644 --- a/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ping_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -298,11 +298,6 @@ processors: - ping_one.audit.actors.client.name ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 5640bd42bd9..91fd2202edb 100644 --- a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -75,11 +75,6 @@ processors: field: "user.email" value: "{{{user.name}}}@{{{user.domain}}}" if: ctx.user?.name != null && ctx.user?.domain != null - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: - _conf diff --git a/packages/prisma_access/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_access/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 07e2594410a..d495d36e51b 100644 --- a/packages/prisma_access/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_access/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -5275,11 +5275,6 @@ processors: field: cef tag: remove_cef ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: description: Drops null/empty values recursively. tag: script_to_drop_null_values diff --git a/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index f1ee8734dc0..92785a5c224 100644 --- a/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -1469,11 +1469,6 @@ processors: ignore_missing: true tag: remove_preserve_duplicate_custom_fields if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless tag: script_painless diff --git a/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 345f6a5f8c0..368a9a73148 100644 --- a/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -190,11 +190,6 @@ processors: field: - json ignore_missing: true - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - remove: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true diff --git a/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml index b13613128c1..2e3b3d0cc5c 100644 --- a/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/host/elasticsearch/ingest_pipeline/default.yml @@ -4220,11 +4220,6 @@ processors: ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) tag: remove_preserve_duplicate_custom_fields - - remove: - field: event.original - ignore_missing: true - tag: remove_event_original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless source: |- diff --git a/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml index 492d8289efc..e23d05f57e6 100644 --- a/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/host_profile/elasticsearch/ingest_pipeline/default.yml @@ -713,11 +713,6 @@ processors: field: - json ignore_missing: true - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - remove: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true diff --git a/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml b/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml index 864ac207491..046a0453ecc 100644 --- a/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/prisma_cloud/data_stream/incident_audit/elasticsearch/ingest_pipeline/default.yml @@ -764,11 +764,6 @@ processors: field: - json ignore_missing: true - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - remove: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true diff --git a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml index 6c12a0111a4..4bffebd4cca 100644 --- a/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_blocked/elasticsearch/ingest_pipeline/default.yml @@ -198,11 +198,6 @@ processors: if: ctx.destination?.ip != null && ctx.destination.ip != '' allow_duplicates: false ignore_failure: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - remove: field: json ignore_missing: true diff --git a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml index 2895ecdd92b..82a6380db28 100644 --- a/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/clicks_permitted/elasticsearch/ingest_pipeline/default.yml @@ -198,11 +198,6 @@ processors: if: ctx.destination?.ip != null && ctx.destination.ip != '' allow_duplicates: false ignore_failure: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - remove: field: json ignore_missing: true diff --git a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml index d501a6e8081..925e1f37b90 100644 --- a/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_blocked/elasticsearch/ingest_pipeline/default.yml @@ -503,11 +503,6 @@ processors: ctx['@timestamp'] = item.threat.time; } } - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - remove: field: - json diff --git a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml index 704b997014e..d237c6ddb54 100644 --- a/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml +++ b/packages/proofpoint_tap/data_stream/message_delivered/elasticsearch/ingest_pipeline/default.yml @@ -482,11 +482,6 @@ processors: ctx['@timestamp'] = item.threat.time; } } - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true - remove: field: - json diff --git a/packages/pulse_connect_secure/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pulse_connect_secure/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 12ccc8cf465..5c3ef100eb8 100644 --- a/packages/pulse_connect_secure/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pulse_connect_secure/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -124,11 +124,6 @@ processors: field: - _tmp ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml index 04118fc774d..1e2d60f3aec 100644 --- a/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_vmdr/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml @@ -113,11 +113,6 @@ processors: - remove: if: ctx?.qualys_vmdr instanceof Map && ctx.qualys_vmdr.isEmpty() field: qualys_vmdr - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: description: Remove metadata unrelated to the user activity log. field: diff --git a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 22c314c4f5e..84a338a4bee 100644 --- a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -698,10 +698,6 @@ processors: field: - _ingest._value.vulnerability_id ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 4b437481b8c..3c5ca51a8cd 100644 --- a/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/rapid7_insightvm/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -425,10 +425,6 @@ processors: - rapid7.insightvm.vulnerability.cvss.v3.score - rapid7.insightvm.vulnerability.severity ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 527f0d71f99..86f70fca73d 100644 --- a/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/santa/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -243,11 +243,6 @@ processors: field: file.x509.issuer.common_name value: "{{{santa.certificate.common_name}}}" if: ctx.santa?.certificate?.common_name != null - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively diff --git a/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml index 0fcbf498bd1..cf2e465345b 100644 --- a/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml @@ -508,11 +508,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/sentinel_one/data_stream/agent/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/agent/elasticsearch/ingest_pipeline/default.yml index bba9855f730..1a94d6e6ab3 100644 --- a/packages/sentinel_one/data_stream/agent/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/agent/elasticsearch/ingest_pipeline/default.yml @@ -796,11 +796,6 @@ processors: ignore_missing: true ignore_failure: true if: ctx.sentinel_one?.agent?.tags != null && ctx.sentinel_one.agent.tags instanceof List - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: - json diff --git a/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 283438c0b34..90c55abe3a3 100644 --- a/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -749,11 +749,6 @@ processors: ignore_missing: true - remove: field: json - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/sentinel_one/data_stream/group/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/group/elasticsearch/ingest_pipeline/default.yml index 0e1300ac04b..9c906b9376a 100644 --- a/packages/sentinel_one/data_stream/group/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/group/elasticsearch/ingest_pipeline/default.yml @@ -141,11 +141,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/sentinel_one/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index e59c35e56a0..c753a44b2cd 100644 --- a/packages/sentinel_one/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -1149,11 +1149,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 402ae95d245..440ed3b3002 100644 --- a/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sentinel_one_cloud_funnel/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -2923,10 +2923,6 @@ processors: - sentinel_one_cloud_funnel.event.url.address if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/slack/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/slack/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 763d19431f8..f8eb4d99e5a 100644 --- a/packages/slack/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/slack/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -436,11 +436,6 @@ processors: } def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/snyk/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/snyk/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 198f726ab31..63df78d6b74 100644 --- a/packages/snyk/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snyk/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -72,11 +72,6 @@ processors: list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - snyk.audit.created diff --git a/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index 7d51dec5889..88999a3fa91 100644 --- a/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snyk/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -150,11 +150,6 @@ processors: list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); } handleMap(ctx); - - remove: - field: event.original - if: ctx?.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: - snyk.audit_logs.created diff --git a/packages/snyk/data_stream/issues/elasticsearch/ingest_pipeline/default.yml b/packages/snyk/data_stream/issues/elasticsearch/ingest_pipeline/default.yml index 8edf4b40907..14674ea8eb7 100644 --- a/packages/snyk/data_stream/issues/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snyk/data_stream/issues/elasticsearch/ingest_pipeline/default.yml @@ -104,11 +104,6 @@ processors: list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); } handleMap(ctx); - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - remove: field: - message diff --git a/packages/snyk/data_stream/vulnerabilities/elasticsearch/ingest_pipeline/default.yml b/packages/snyk/data_stream/vulnerabilities/elasticsearch/ingest_pipeline/default.yml index eedeeb3f4fe..2e500af5480 100644 --- a/packages/snyk/data_stream/vulnerabilities/elasticsearch/ingest_pipeline/default.yml +++ b/packages/snyk/data_stream/vulnerabilities/elasticsearch/ingest_pipeline/default.yml @@ -186,11 +186,6 @@ processors: list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0)); } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - message diff --git a/packages/sophos_central/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/sophos_central/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 0dc9b369661..e44945287bf 100644 --- a/packages/sophos_central/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sophos_central/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -628,10 +628,6 @@ processors: - sophos_central.alert.data.ips_threat.remote.ip - sophos_central.alert.data.ips_threat.remote.port ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/sophos_central/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/sophos_central/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 44738d02a0d..fb0ae80b77d 100644 --- a/packages/sophos_central/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sophos_central/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -420,10 +420,6 @@ processors: - sophos_central.event.ips_threat_data.remote.port - sophos_central.event.when ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/symantec_edr_cloud/data_stream/incident/elasticsearch/ingest_pipeline/default.yml b/packages/symantec_edr_cloud/data_stream/incident/elasticsearch/ingest_pipeline/default.yml index de549906018..3d34afd1d2b 100644 --- a/packages/symantec_edr_cloud/data_stream/incident/elasticsearch/ingest_pipeline/default.yml +++ b/packages/symantec_edr_cloud/data_stream/incident/elasticsearch/ingest_pipeline/default.yml @@ -433,11 +433,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/symantec_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/symantec_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml index eef279f5f8b..9e2855f5ae5 100644 --- a/packages/symantec_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/symantec_endpoint/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1097,12 +1097,6 @@ processors: ctx.source = ctx.destination; ctx.destination = tmp; -- remove: - description: Retain event.original when preserve_original_event tag exists. - if: ctx.tags == null || !ctx.tags.contains('preserve_original_event') - field: event.original - ignore_missing: true - - remove: if: ctx.tags == null || !ctx.tags.contains('debug') ignore_missing: true diff --git a/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml index ec756b2eaba..8e046892262 100644 --- a/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/symantec_endpoint_security/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -1197,11 +1197,6 @@ processors: tag: remove_custom_duplicate_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/symantec_endpoint_security/data_stream/incident/elasticsearch/ingest_pipeline/default.yml b/packages/symantec_endpoint_security/data_stream/incident/elasticsearch/ingest_pipeline/default.yml index e5ba7e0f59a..8000ee186bc 100644 --- a/packages/symantec_endpoint_security/data_stream/incident/elasticsearch/ingest_pipeline/default.yml +++ b/packages/symantec_endpoint_security/data_stream/incident/elasticsearch/ingest_pipeline/default.yml @@ -427,11 +427,6 @@ processors: tag: remove_duplicate_custom_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/tanium/data_stream/action_history/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/action_history/elasticsearch/ingest_pipeline/default.yml index 33be197ecfa..9b15a63e641 100644 --- a/packages/tanium/data_stream/action_history/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/action_history/elasticsearch/ingest_pipeline/default.yml @@ -176,11 +176,6 @@ processors: - tanium.action_history.action.name - tanium.action_history.command ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: tag: script_drops_null_empty_values_recursively description: Drops null/empty values recursively. diff --git a/packages/tanium/data_stream/client_status/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/client_status/elasticsearch/ingest_pipeline/default.yml index b155864ad3d..dcfa2e7b8ba 100644 --- a/packages/tanium/data_stream/client_status/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/client_status/elasticsearch/ingest_pipeline/default.yml @@ -189,11 +189,6 @@ processors: - tanium.client_status.client_network_location - tanium.client_status.server_network_location ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: tag: script_drops_null_empty_values_recursively description: Drops null/empty values recursively. diff --git a/packages/tanium/data_stream/discover/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/discover/elasticsearch/ingest_pipeline/default.yml index 8f3840fd6e8..d4b2d23edc0 100644 --- a/packages/tanium/data_stream/discover/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/discover/elasticsearch/ingest_pipeline/default.yml @@ -391,11 +391,6 @@ processors: tag: remove_json field: json ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - remove: tag: remove_duplicate_custom_fields if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) diff --git a/packages/tanium/data_stream/endpoint_config/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/endpoint_config/elasticsearch/ingest_pipeline/default.yml index f548adf1d8a..f2c3f823b95 100644 --- a/packages/tanium/data_stream/endpoint_config/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/endpoint_config/elasticsearch/ingest_pipeline/default.yml @@ -214,11 +214,6 @@ processors: - tanium.endpoint_config.action - tanium.endpoint_config.user.id ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: tag: script_drops_null_empty_values_recursively description: Drops null/empty values recursively. diff --git a/packages/tanium/data_stream/reporting/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/reporting/elasticsearch/ingest_pipeline/default.yml index 1585cc87a66..4c29e4acc7f 100644 --- a/packages/tanium/data_stream/reporting/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/reporting/elasticsearch/ingest_pipeline/default.yml @@ -114,11 +114,6 @@ processors: - tanium.reporting.os.name - tanium.reporting.computer_name ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: tag: script_drops_null_empty_values_recursively description: Drops null/empty values recursively. diff --git a/packages/tanium/data_stream/threat_response/elasticsearch/ingest_pipeline/default.yml b/packages/tanium/data_stream/threat_response/elasticsearch/ingest_pipeline/default.yml index 4a21455e043..3317c96ccab 100644 --- a/packages/tanium/data_stream/threat_response/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tanium/data_stream/threat_response/elasticsearch/ingest_pipeline/default.yml @@ -363,11 +363,6 @@ processors: tag: remove_json field: json ignore_missing: true - - remove: - tag: remove_event_original - field: event.original - if: ctx.tags == null || !(ctx.tags.contains("preserve_original_event")) - ignore_missing: true - remove: tag: remove_duplicate_custom_fields if: ctx.tags == null || !(ctx.tags.contains("preserve_duplicate_custom_fields")) diff --git a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 6e72624f2ed..3693bc502d8 100644 --- a/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/teleport/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -127,12 +127,6 @@ processors: name: '{{ IngestPipeline "event-enrich" }}' ignore_failure: true # Final steps. - - remove: - field: event.original - tag: remove_original_event - if: ctx?.tags == null || !(ctx.tags.contains("preserve_original_event")) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. tag: script_drop_null_empty_values diff --git a/packages/tenable_io/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index b649c2f5757..b6700f6fbd5 100644 --- a/packages/tenable_io/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_io/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -706,10 +706,6 @@ processors: - tenable_io.asset.operating_systems - tenable_io.asset.network.name ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml index 5398c9d23a2..61248aa20e0 100644 --- a/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml @@ -559,10 +559,6 @@ processors: - tenable_io.plugin.attributes.cvss3.base_score - tenable_io.plugin.attributes.cvss3.temporal.score ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/tenable_io/data_stream/scan/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/scan/elasticsearch/ingest_pipeline/default.yml index a92aa095728..ba0b0f1edfc 100644 --- a/packages/tenable_io/data_stream/scan/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_io/data_stream/scan/elasticsearch/ingest_pipeline/default.yml @@ -39,10 +39,6 @@ processors: if: ctx.json?.starttime != null && ctx.json.starttime != '' formats: - yyyyMMdd'T'HHmmss - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - rename: field: json target_field: tenable_io.scan diff --git a/packages/tenable_io/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 278732db36e..22adfb372dd 100644 --- a/packages/tenable_io/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_io/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -1003,10 +1003,6 @@ processors: - tenable_io.vulnerability.asset.ip_address - tenable_io.vulnerability.asset.hostname ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/tenable_sc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_sc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml index 2c572a0193b..d3cf92c6a7d 100644 --- a/packages/tenable_sc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_sc/data_stream/asset/elasticsearch/ingest_pipeline/default.yml @@ -281,11 +281,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/tenable_sc/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_sc/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml index c00b6610cc2..504404f2f78 100644 --- a/packages/tenable_sc/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_sc/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml @@ -403,11 +403,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/tenable_sc/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_sc/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index ad33843dce4..0b8255d51da 100644 --- a/packages/tenable_sc/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_sc/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -581,11 +581,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags?.contains('preserve_original_event') != true - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/thycotic_ss/data_stream/logs/elasticsearch/ingest_pipeline/default.yml b/packages/thycotic_ss/data_stream/logs/elasticsearch/ingest_pipeline/default.yml index d4cb7cf86d5..ccf87f65826 100644 --- a/packages/thycotic_ss/data_stream/logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/thycotic_ss/data_stream/logs/elasticsearch/ingest_pipeline/default.yml @@ -437,13 +437,6 @@ processors: ## Cleanup Fields ## #################### - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - description: "Remove event.original unless tags indicate we shold not" - - remove: field: - cef diff --git a/packages/ti_abusech/data_stream/malware/elasticsearch/ingest_pipeline/default.yml b/packages/ti_abusech/data_stream/malware/elasticsearch/ingest_pipeline/default.yml index cb2df0b6d92..8f794a400ac 100644 --- a/packages/ti_abusech/data_stream/malware/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_abusech/data_stream/malware/elasticsearch/ingest_pipeline/default.yml @@ -200,11 +200,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - abusech.malware.firstseen diff --git a/packages/ti_abusech/data_stream/malwarebazaar/elasticsearch/ingest_pipeline/default.yml b/packages/ti_abusech/data_stream/malwarebazaar/elasticsearch/ingest_pipeline/default.yml index c00c3ca24fc..7f9430a362e 100644 --- a/packages/ti_abusech/data_stream/malwarebazaar/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_abusech/data_stream/malwarebazaar/elasticsearch/ingest_pipeline/default.yml @@ -316,11 +316,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - abusech.malwarebazaar.first_seen diff --git a/packages/ti_abusech/data_stream/threatfox/elasticsearch/ingest_pipeline/default.yml b/packages/ti_abusech/data_stream/threatfox/elasticsearch/ingest_pipeline/default.yml index a888ce82ba6..d7de8c0603c 100644 --- a/packages/ti_abusech/data_stream/threatfox/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_abusech/data_stream/threatfox/elasticsearch/ingest_pipeline/default.yml @@ -244,11 +244,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - abusech.threatfox.first_seen diff --git a/packages/ti_abusech/data_stream/url/elasticsearch/ingest_pipeline/default.yml b/packages/ti_abusech/data_stream/url/elasticsearch/ingest_pipeline/default.yml index c3f8c87e4a4..cb4d13bf61e 100644 --- a/packages/ti_abusech/data_stream/url/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_abusech/data_stream/url/elasticsearch/ingest_pipeline/default.yml @@ -194,10 +194,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_missing: true - remove: field: - abusech.url.date_added diff --git a/packages/ti_anomali/data_stream/threatstream/elasticsearch/ingest_pipeline/default.yml b/packages/ti_anomali/data_stream/threatstream/elasticsearch/ingest_pipeline/default.yml index 75619f5ee0e..12e97337318 100644 --- a/packages/ti_anomali/data_stream/threatstream/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_anomali/data_stream/threatstream/elasticsearch/ingest_pipeline/default.yml @@ -479,11 +479,6 @@ processors: # # Remove fields converted to an ECS field. # - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - _temp_ diff --git a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml index 70dbb991ad8..7805fdbc87d 100644 --- a/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cif3/data_stream/feed/elasticsearch/ingest_pipeline/default.yml @@ -407,11 +407,6 @@ processors: field: cif3.rdata ignore_missing: true if: "ctx.cif3?.rdata == ''" - - remove: - field: event.original - if: "ctx.tags?.contains('preserve_original_event') != true" - ignore_failure: true - ignore_missing: true - remove: field: - cif3.confidence diff --git a/packages/ti_crowdstrike/data_stream/intel/elasticsearch/ingest_pipeline/default.yml b/packages/ti_crowdstrike/data_stream/intel/elasticsearch/ingest_pipeline/default.yml index 5274e43d213..335d08bdfac 100644 --- a/packages/ti_crowdstrike/data_stream/intel/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_crowdstrike/data_stream/intel/elasticsearch/ingest_pipeline/default.yml @@ -416,11 +416,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_crowdstrike/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml b/packages/ti_crowdstrike/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml index 8b49c07da41..afcc17ecada 100644 --- a/packages/ti_crowdstrike/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_crowdstrike/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml @@ -337,11 +337,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_cybersixgill/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_cybersixgill/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index aee01d92833..aaa75648bf4 100644 --- a/packages/ti_cybersixgill/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_cybersixgill/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -325,11 +325,6 @@ processors: } } handleMap(ctx); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - _temp_ diff --git a/packages/ti_eset/data_stream/apt/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/apt/elasticsearch/ingest_pipeline/default.yml index 084fb717f5c..c12af671235 100644 --- a/packages/ti_eset/data_stream/apt/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/apt/elasticsearch/ingest_pipeline/default.yml @@ -128,11 +128,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/botnet/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/botnet/elasticsearch/ingest_pipeline/default.yml index cf5aee06912..97d2346f3f2 100644 --- a/packages/ti_eset/data_stream/botnet/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/botnet/elasticsearch/ingest_pipeline/default.yml @@ -118,11 +118,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/cc/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/cc/elasticsearch/ingest_pipeline/default.yml index 8809216de63..3e87331d946 100644 --- a/packages/ti_eset/data_stream/cc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/cc/elasticsearch/ingest_pipeline/default.yml @@ -110,11 +110,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/domains/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/domains/elasticsearch/ingest_pipeline/default.yml index 008c8ab9b3a..ff6fc67ea80 100644 --- a/packages/ti_eset/data_stream/domains/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/domains/elasticsearch/ingest_pipeline/default.yml @@ -114,11 +114,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/files/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/files/elasticsearch/ingest_pipeline/default.yml index f8d7bff278d..50b65063ba3 100644 --- a/packages/ti_eset/data_stream/files/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/files/elasticsearch/ingest_pipeline/default.yml @@ -111,11 +111,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/ip/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/ip/elasticsearch/ingest_pipeline/default.yml index 64c37434dd4..87f8e1de767 100644 --- a/packages/ti_eset/data_stream/ip/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/ip/elasticsearch/ingest_pipeline/default.yml @@ -111,11 +111,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_eset/data_stream/url/elasticsearch/ingest_pipeline/default.yml b/packages/ti_eset/data_stream/url/elasticsearch/ingest_pipeline/default.yml index f60da66bd89..316d0ea8860 100644 --- a/packages/ti_eset/data_stream/url/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_eset/data_stream/url/elasticsearch/ingest_pipeline/default.yml @@ -109,11 +109,6 @@ processors: field: - eti ignore_missing: true - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_maltiverse/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_maltiverse/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 80c847054f8..ed03e750a07 100644 --- a/packages/ti_maltiverse/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_maltiverse/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -266,11 +266,6 @@ processors: value: "{{{ event.id }}}" if: 'ctx.event?.id != null' - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - remove: field: - maltiverse.tag diff --git a/packages/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index f818a5ed41e..d777dbb18a6 100644 --- a/packages/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -438,11 +438,6 @@ processors: } handleMap(ctx); # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - misp.attribute.value diff --git a/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml b/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml index 58a40be8f1e..ce76e268e56 100644 --- a/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_misp/data_stream/threat_attributes/elasticsearch/ingest_pipeline/default.yml @@ -531,12 +531,6 @@ processors: } handleMap(ctx); # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - - remove: field: - misp.attribute.value diff --git a/packages/ti_otx/data_stream/pulses_subscribed/elasticsearch/ingest_pipeline/default.yml b/packages/ti_otx/data_stream/pulses_subscribed/elasticsearch/ingest_pipeline/default.yml index 5342ebe6efd..65e8910b4f6 100644 --- a/packages/ti_otx/data_stream/pulses_subscribed/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_otx/data_stream/pulses_subscribed/elasticsearch/ingest_pipeline/default.yml @@ -317,11 +317,6 @@ processors: - otx.pulse.adversary ignore_missing: true if: ctx.otx?.pulse?.adversary == "" - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - otx.type diff --git a/packages/ti_otx/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_otx/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index b36ed839d03..c01b91feeab 100644 --- a/packages/ti_otx/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_otx/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -192,11 +192,6 @@ processors: - otx.content ignore_missing: true if: ctx.otx?.content == "" - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - otx.type diff --git a/packages/ti_rapid7_threat_command/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/ti_rapid7_threat_command/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index c72f829cb2c..3c9a9b19264 100644 --- a/packages/ti_rapid7_threat_command/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_rapid7_threat_command/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -214,11 +214,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - ignore_failure: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/ti_rapid7_threat_command/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml b/packages/ti_rapid7_threat_command/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml index c8b956c231d..76e893a7161 100644 --- a/packages/ti_rapid7_threat_command/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_rapid7_threat_command/data_stream/ioc/elasticsearch/ingest_pipeline/default.yml @@ -424,11 +424,6 @@ processors: } drop(ctx); ignore_failure: true - - remove: - field: event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - fingerprint: fields: - rapid7.tc.ioc.last_update_date diff --git a/packages/ti_rapid7_threat_command/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml b/packages/ti_rapid7_threat_command/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml index 90f58923bb8..93f31249c72 100644 --- a/packages/ti_rapid7_threat_command/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_rapid7_threat_command/data_stream/vulnerability/elasticsearch/ingest_pipeline/default.yml @@ -311,11 +311,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - ignore_failure: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index 2d837174c90..06ceb5552da 100644 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -343,11 +343,6 @@ processors: # # Cleanup # - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - recordedfuture.Algorithm diff --git a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 4bc6bf26dfe..4b9b2c53943 100644 --- a/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatconnect/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -2198,11 +2198,6 @@ processors: field: json tag: remove_json ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/ti_threatq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_threatq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index 3e0802a45e3..a74641b429a 100644 --- a/packages/ti_threatq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_threatq/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -427,11 +427,6 @@ processors: handleMap(ctx); # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event - - remove: - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - remove: field: - json diff --git a/packages/tines/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/tines/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index af688bd0946..6e5d5893a83 100644 --- a/packages/tines/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tines/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -184,11 +184,6 @@ processors: - message - json ignore_missing: true - - remove: - description: "Remove event.original unless tags indicate not to" - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_missing: true - remove: description: "Remove ECS mapped field original duplicates unless tags indicate not to" field: diff --git a/packages/tines/data_stream/time_saved/elasticsearch/ingest_pipeline/default.yml b/packages/tines/data_stream/time_saved/elasticsearch/ingest_pipeline/default.yml index c0a60bd859b..e83c6e7f192 100644 --- a/packages/tines/data_stream/time_saved/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tines/data_stream/time_saved/elasticsearch/ingest_pipeline/default.yml @@ -48,11 +48,6 @@ processors: - message - json ignore_missing: true - - remove: - description: "Remove event.original unless tags indicate not to" - field: event.original - if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_missing: true - remove: description: "Remove ECS mapped field original duplicates unless tags indicate not to" field: diff --git a/packages/trellix_edr_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/trellix_edr_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 0cc556fcaa8..6f9aff932d4 100644 --- a/packages/trellix_edr_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trellix_edr_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -901,11 +901,6 @@ processors: tag: remove_preserve_duplicate_custom_fields if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_missing: true - script: lang: painless tag: script_to_remove_null_values diff --git a/packages/trellix_epo_cloud/data_stream/device/elasticsearch/ingest_pipeline/default.yml b/packages/trellix_epo_cloud/data_stream/device/elasticsearch/ingest_pipeline/default.yml index 0ad6dfeec31..3e560997b88 100644 --- a/packages/trellix_epo_cloud/data_stream/device/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trellix_epo_cloud/data_stream/device/elasticsearch/ingest_pipeline/default.yml @@ -454,11 +454,6 @@ processors: - trellix_epo_cloud.device.attributes.user_name ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/trellix_epo_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/trellix_epo_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml index 24b68f3baa5..d6577862916 100644 --- a/packages/trellix_epo_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trellix_epo_cloud/data_stream/event/elasticsearch/ingest_pipeline/default.yml @@ -631,11 +631,6 @@ processors: - trellix_epo_cloud.event.attributes.threat.action_taken ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/trellix_epo_cloud/data_stream/group/elasticsearch/ingest_pipeline/default.yml b/packages/trellix_epo_cloud/data_stream/group/elasticsearch/ingest_pipeline/default.yml index 6ece3410e01..819b42ddca3 100644 --- a/packages/trellix_epo_cloud/data_stream/group/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trellix_epo_cloud/data_stream/group/elasticsearch/ingest_pipeline/default.yml @@ -151,11 +151,6 @@ processors: - trellix_epo_cloud.group.attributes.name ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: - - event.original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: lang: painless description: Drops null/empty values recursively. diff --git a/packages/trend_micro_vision_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/trend_micro_vision_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml index 339006866ec..bcc3baa8419 100644 --- a/packages/trend_micro_vision_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trend_micro_vision_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml @@ -575,11 +575,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/trend_micro_vision_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/trend_micro_vision_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 84a44733b50..3d8d4ad36e4 100644 --- a/packages/trend_micro_vision_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trend_micro_vision_one/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -146,11 +146,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/trend_micro_vision_one/data_stream/detection/elasticsearch/ingest_pipeline/default.yml b/packages/trend_micro_vision_one/data_stream/detection/elasticsearch/ingest_pipeline/default.yml index 78cd5c42b07..b2984f4e515 100644 --- a/packages/trend_micro_vision_one/data_stream/detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trend_micro_vision_one/data_stream/detection/elasticsearch/ingest_pipeline/default.yml @@ -969,11 +969,6 @@ processors: if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) ignore_failure: true ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true - script: description: Drops null/empty values recursively. lang: painless diff --git a/packages/trendmicro/data_stream/deep_security/elasticsearch/ingest_pipeline/default.yml b/packages/trendmicro/data_stream/deep_security/elasticsearch/ingest_pipeline/default.yml index 52aa14185fc..530775ee5de 100644 --- a/packages/trendmicro/data_stream/deep_security/elasticsearch/ingest_pipeline/default.yml +++ b/packages/trendmicro/data_stream/deep_security/elasticsearch/ingest_pipeline/default.yml @@ -959,11 +959,6 @@ processors: - cef tag: remove_cef_and_tmp ignore_missing: true - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: tag: script_to_drop_null_values lang: painless diff --git a/packages/vectra_detect/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/vectra_detect/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 91ec0c96b4d..0b1cbf65a47 100644 --- a/packages/vectra_detect/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/vectra_detect/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -143,10 +143,6 @@ processors: - json - _tmp ignore_missing: true - - remove: - field: event.original - ignore_missing: true - if: ctx.tags?.contains('preserve_original_event') != true - remove: if: ctx.tags?.contains('preserve_duplicate_custom_fields') != true field: diff --git a/packages/wiz/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/wiz/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 013d9549e4f..7ba50c49bf0 100644 --- a/packages/wiz/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/wiz/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -282,11 +282,6 @@ processors: tag: remove_custom_duplicate_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: description: Drops null/empty values recursively. tag: script_to_drop_null_values diff --git a/packages/wiz/data_stream/issue/elasticsearch/ingest_pipeline/default.yml b/packages/wiz/data_stream/issue/elasticsearch/ingest_pipeline/default.yml index 8e63254d270..f5a1aefefbf 100644 --- a/packages/wiz/data_stream/issue/elasticsearch/ingest_pipeline/default.yml +++ b/packages/wiz/data_stream/issue/elasticsearch/ingest_pipeline/default.yml @@ -394,11 +394,6 @@ processors: tag: remove_custom_duplicate_fields ignore_missing: true if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields')) - - remove: - field: event.original - tag: remove_event_original - ignore_missing: true - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - script: description: Drops null/empty values recursively. tag: script_to_drop_null_values diff --git a/packages/zerofox/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/zerofox/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 6aed1e10ec3..87342818b62 100644 --- a/packages/zerofox/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zerofox/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -176,11 +176,6 @@ processors: lang: painless if: ctx?.zerofox != null source: ctx?.zerofox?.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue().equals("") || (entry.getValue() instanceof List && entry.getValue().length == 0) || (entry.getValue() instanceof Map && entry.getValue().size() == 0)); - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/zeronetworks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/zeronetworks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index d0746b4df36..5e71bea4fc3 100644 --- a/packages/zeronetworks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeronetworks/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -1109,11 +1109,6 @@ processors: field: - json ignore_missing: true - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true - script: lang: painless description: This script processor iterates over the whole document to remove fields with null values. diff --git a/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml index ff2c108167f..bcaf90eae2a 100644 --- a/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/app_connector_status/elasticsearch/ingest_pipeline/default.yml @@ -353,11 +353,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index 3173230f5dc..7315d7d54be 100644 --- a/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -301,11 +301,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml index 3c146e4b6b1..2591194ace2 100644 --- a/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/browser_access/elasticsearch/ingest_pipeline/default.yml @@ -408,11 +408,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml index 0ffacf38e9c..d6b20f60387 100644 --- a/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/user_activity/elasticsearch/ingest_pipeline/default.yml @@ -550,11 +550,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind diff --git a/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml index 6122bac9cc1..d95717abdd3 100644 --- a/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zpa/data_stream/user_status/elasticsearch/ingest_pipeline/default.yml @@ -270,11 +270,6 @@ processors: - remove: field: json ignore_missing: true - - remove: - field: event.original - if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event')) - ignore_failure: true - ignore_missing: true on_failure: - set: field: event.kind