From 2621bb092296a4773be9796cf95a069eefa7b2f4 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 9 Oct 2024 09:52:30 +1030 Subject: [PATCH] sdd_all: use triple-brace templating in complex templates (#11315) This is a second pass at replacing double brace template marks with triple brace marks. It addresses more complex uses that the sed rewrite did not find by using PCRE via perl. [git-generate] for f in $( ( for p in $( yq 'select(.owner.github == "elastic/sec-deployment-and-devices")|.name' packages/**/manifest.yml \ | grep -v -- '---' ); do rg -l -g '*.yml' "[^{]\{\{[^{][ .a-zA-Z0-9_]*[^}]}}[^}]" packages/$p done )|grep "elasticsearch/ingest_pipeline"|sort|uniq ); do perl -pi -e 's/(?/dev/null 2>&1 done --- packages/cef/changelog.yml | 5 +++ .../ingest_pipeline/cp-pipeline.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../ingest_pipeline/fp-pipeline.yml | 2 +- packages/cef/manifest.yml | 2 +- packages/checkpoint/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 10 ++--- packages/checkpoint/manifest.yml | 2 +- packages/cisco_ftd/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 4 +- packages/cisco_ftd/manifest.yml | 2 +- packages/cisco_ios/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 2 +- packages/cisco_ios/manifest.yml | 2 +- packages/cisco_ise/changelog.yml | 5 +++ .../pipeline_passed_authentications.yml | 6 +-- packages/cisco_ise/manifest.yml | 2 +- .../cisco_secure_email_gateway/changelog.yml | 5 +++ .../pipeline_consolidated_event.yml | 2 +- .../cisco_secure_email_gateway/manifest.yml | 2 +- packages/goflow2/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 6 +-- packages/goflow2/manifest.yml | 2 +- packages/iptables/changelog.yml | 5 +++ .../test-iptables-raw.log-expected.json | 42 +++++++++---------- .../pipeline/test-ubiquiti.log-expected.json | 22 +++++----- .../elasticsearch/ingest_pipeline/default.yml | 6 +-- packages/iptables/manifest.yml | 2 +- packages/juniper_srx/changelog.yml | 5 +++ .../log/elasticsearch/ingest_pipeline/atp.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../elasticsearch/ingest_pipeline/flow.yml | 2 +- .../log/elasticsearch/ingest_pipeline/idp.yml | 2 +- .../log/elasticsearch/ingest_pipeline/ids.yml | 2 +- .../ingest_pipeline/secintel.yml | 2 +- .../elasticsearch/ingest_pipeline/system.yml | 22 +++++----- .../log/elasticsearch/ingest_pipeline/utm.yml | 2 +- packages/juniper_srx/manifest.yml | 2 +- packages/modsecurity/changelog.yml | 5 +++ .../ingest_pipeline/apache-modsec.yml | 6 +-- .../ingest_pipeline/nginx-modsec.yml | 6 +-- packages/modsecurity/manifest.yml | 2 +- packages/pfsense/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 2 +- packages/pfsense/manifest.yml | 2 +- packages/sophos/changelog.yml | 5 +++ .../elasticsearch/ingest_pipeline/default.yml | 6 +-- .../elasticsearch/ingest_pipeline/dhcp.yml | 2 +- .../utm/elasticsearch/ingest_pipeline/dns.yml | 2 +- .../elasticsearch/ingest_pipeline/http.yml | 2 +- .../ingest_pipeline/packetfilter.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 6 +-- packages/sophos/manifest.yml | 2 +- packages/zeek/changelog.yml | 5 +++ .../ingest_pipeline/third-party.yml | 6 +-- packages/zeek/manifest.yml | 2 +- 56 files changed, 169 insertions(+), 104 deletions(-) diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml index bbd8b7de109..397d0a45e93 100644 --- a/packages/cef/changelog.yml +++ b/packages/cef/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.17.4" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "2.17.3" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml index cfdf9e1c389..1004bdb089a 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml @@ -379,7 +379,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 3770561d67f..375eebc67ab 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -197,7 +197,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml index 7e51daff711..760751bdd91 100644 --- a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml +++ b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml @@ -25,7 +25,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/cef/manifest.yml b/packages/cef/manifest.yml index 32660a65ccc..84e184448fe 100644 --- a/packages/cef/manifest.yml +++ b/packages/cef/manifest.yml @@ -1,6 +1,6 @@ name: cef title: Common Event Format (CEF) -version: "2.17.3" +version: "2.17.4" description: Collect logs from CEF Logs with Elastic Agent. categories: - security diff --git a/packages/checkpoint/changelog.yml b/packages/checkpoint/changelog.yml index c5eb23adb5c..c20f7eb7335 100644 --- a/packages/checkpoint/changelog.yml +++ b/packages/checkpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.34.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.34.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml index 75ef37fe2fa..d99435f24b7 100644 --- a/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml +++ b/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml @@ -28,7 +28,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - kv: tag: "kv_syslog_structured_semicolon_colon" field: syslog5424_sd @@ -52,7 +52,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - grok: field: syslog5424_sd if: ctx.checkpoint == null @@ -86,7 +86,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - foreach: field: checkpoint ignore_missing: true @@ -160,7 +160,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # Some log events lack loguid and time, so to avoid potential # collisions hash the complete line in those rare cases. - fingerprint: @@ -1066,7 +1066,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: checkpoint._temp_unixms target_field: "@timestamp" diff --git a/packages/checkpoint/manifest.yml b/packages/checkpoint/manifest.yml index 2b1626bbbef..683de00d3b2 100644 --- a/packages/checkpoint/manifest.yml +++ b/packages/checkpoint/manifest.yml @@ -1,6 +1,6 @@ name: checkpoint title: Check Point -version: "1.34.1" +version: "1.34.2" description: Collect logs from Check Point with Elastic Agent. type: integration format_version: "3.0.3" diff --git a/packages/cisco_ftd/changelog.yml b/packages/cisco_ftd/changelog.yml index 8fcb95c4863..6aba51aed19 100644 --- a/packages/cisco_ftd/changelog.yml +++ b/packages/cisco_ftd/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.4.3" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "3.4.2" changes: - description: "Fix grok failure with username with spaces on ftd messageID." diff --git a/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 74a2ef898ea..f3380a12f94 100644 --- a/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1815,7 +1815,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - grok: field: "destination.user.name" tag: "grok_destination_user_name" @@ -1833,7 +1833,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # # Normalize protocol names diff --git a/packages/cisco_ftd/manifest.yml b/packages/cisco_ftd/manifest.yml index 6df17794421..ec909e3635d 100644 --- a/packages/cisco_ftd/manifest.yml +++ b/packages/cisco_ftd/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_ftd title: Cisco FTD -version: "3.4.2" +version: "3.4.3" description: Collect logs from Cisco FTD with Elastic Agent. type: integration categories: diff --git a/packages/cisco_ios/changelog.yml b/packages/cisco_ios/changelog.yml index 1ae673571e8..da20d9f899c 100644 --- a/packages/cisco_ios/changelog.yml +++ b/packages/cisco_ios/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.27.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.27.1" changes: - description: Handle timestamp starting with the year such as 'yyyy MMM d HH:mm:ss.SSS z' diff --git a/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 48a44fb4d26..563b4d84dd8 100644 --- a/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -461,7 +461,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/cisco_ios/manifest.yml b/packages/cisco_ios/manifest.yml index d711b85e1a1..6c766362f26 100644 --- a/packages/cisco_ios/manifest.yml +++ b/packages/cisco_ios/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_ios title: Cisco IOS -version: "1.27.1" +version: "1.27.2" description: Collect logs from Cisco IOS with Elastic Agent. type: integration categories: diff --git a/packages/cisco_ise/changelog.yml b/packages/cisco_ise/changelog.yml index e4ebfdc7551..ad15bceb0b5 100644 --- a/packages/cisco_ise/changelog.yml +++ b/packages/cisco_ise/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.23.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.23.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml b/packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml index a3533fdcefc..8554a508d99 100644 --- a/packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml +++ b/packages/cisco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml @@ -575,6 +575,6 @@ on_failure: - append: field: error.message value: >- - Processor '{{ _ingest.on_failure_processor_type }}' - {{#_ingest.on_failure_processor_tag}}with tag '{{ _ingest.on_failure_processor_tag }}' - {{/_ingest.on_failure_processor_tag}}failed with message '{{ _ingest.on_failure_message }}' + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' diff --git a/packages/cisco_ise/manifest.yml b/packages/cisco_ise/manifest.yml index 3d7b1f9d468..f8b93fee580 100644 --- a/packages/cisco_ise/manifest.yml +++ b/packages/cisco_ise/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_ise title: Cisco ISE -version: "1.23.1" +version: "1.23.2" description: Collect logs from Cisco ISE with Elastic Agent. type: integration categories: diff --git a/packages/cisco_secure_email_gateway/changelog.yml b/packages/cisco_secure_email_gateway/changelog.yml index d8706a86366..7314d18cf30 100644 --- a/packages/cisco_secure_email_gateway/changelog.yml +++ b/packages/cisco_secure_email_gateway/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.24.1" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.24.0" changes: - description: "Allow @custom pipeline access to event.original without setting preserve_original_event." diff --git a/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml b/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml index 07a6491d63a..548f1e70cc9 100644 --- a/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml +++ b/packages/cisco_secure_email_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml @@ -534,4 +534,4 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" diff --git a/packages/cisco_secure_email_gateway/manifest.yml b/packages/cisco_secure_email_gateway/manifest.yml index 3bff58ecdc5..85dd66dff9c 100644 --- a/packages/cisco_secure_email_gateway/manifest.yml +++ b/packages/cisco_secure_email_gateway/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: cisco_secure_email_gateway title: Cisco Secure Email Gateway -version: "1.24.0" +version: "1.24.1" description: Collect logs from Cisco Secure Email Gateway with Elastic Agent. type: integration categories: diff --git a/packages/goflow2/changelog.yml b/packages/goflow2/changelog.yml index 5cc386f1d8b..1585baf2cb5 100644 --- a/packages/goflow2/changelog.yml +++ b/packages/goflow2/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.1" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "0.1.0" changes: - description: Initial version of the package diff --git a/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml b/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml index fad54500462..a09c693b5f9 100644 --- a/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml +++ b/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml @@ -271,7 +271,7 @@ processors: on_failure: - set: field: error.message - value: Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag - }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message - }} - Source {{_source}} + value: Processor {{{ _ingest.on_failure_processor_type }}} with tag {{ _ingest.on_failure_processor_tag + }} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message {{ _ingest.on_failure_message + }} - Source {{{_source}}} tag: set_error_message_on_failure diff --git a/packages/goflow2/manifest.yml b/packages/goflow2/manifest.yml index b2d8af2fe8a..d38c299d765 100644 --- a/packages/goflow2/manifest.yml +++ b/packages/goflow2/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.1 name: goflow2 title: "GoFlow2 logs" -version: 0.1.0 +version: 0.1.1 description: "Collect logs from goflow2 with Elastic Agent." type: integration categories: diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml index d75fd2d7bdf..04d1ea401ae 100644 --- a/packages/iptables/changelog.yml +++ b/packages/iptables/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.16.1" changes: - description: Invoke community_id processor only for supported protocols diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json index 3850f654a27..702a622463d 100644 --- a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json +++ b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2023-10-10T07:25:12.000Z", + "@timestamp": "2024-10-10T07:25:12.000Z", "destination": { "ip": "10.4.0.5", "mac": "90-10-20-76-8D-20", @@ -296,7 +296,7 @@ ] }, { - "@timestamp": "2023-01-08T03:37:09.000Z", + "@timestamp": "2024-01-08T03:37:09.000Z", "destination": { "ip": "192.168.2.83", "mac": "90-10-28-5F-62-24" @@ -352,7 +352,7 @@ ] }, { - "@timestamp": "2023-01-08T03:37:09.000Z", + "@timestamp": "2024-01-08T03:37:09.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -431,7 +431,7 @@ ] }, { - "@timestamp": "2023-01-08T03:37:57.000Z", + "@timestamp": "2024-01-08T03:37:57.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -495,7 +495,7 @@ ] }, { - "@timestamp": "2023-01-08T03:38:45.000Z", + "@timestamp": "2024-01-08T03:38:45.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -574,7 +574,7 @@ ] }, { - "@timestamp": "2023-01-08T03:39:25.000Z", + "@timestamp": "2024-01-08T03:39:25.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -653,7 +653,7 @@ ] }, { - "@timestamp": "2023-01-08T03:40:21.000Z", + "@timestamp": "2024-01-08T03:40:21.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -732,7 +732,7 @@ ] }, { - "@timestamp": "2023-01-08T03:40:25.000Z", + "@timestamp": "2024-01-08T03:40:25.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -796,7 +796,7 @@ ] }, { - "@timestamp": "2023-01-08T03:41:17.000Z", + "@timestamp": "2024-01-08T03:41:17.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -863,7 +863,7 @@ ] }, { - "@timestamp": "2023-01-08T03:41:23.000Z", + "@timestamp": "2024-01-08T03:41:23.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -930,7 +930,7 @@ ] }, { - "@timestamp": "2023-01-08T03:43:18.000Z", + "@timestamp": "2024-01-08T03:43:18.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -994,7 +994,7 @@ ] }, { - "@timestamp": "2023-01-08T03:43:42.000Z", + "@timestamp": "2024-01-08T03:43:42.000Z", "destination": { "ip": "172.16.54.114", "mac": "90-10-35-5A-1E-3A", @@ -1058,7 +1058,7 @@ ] }, { - "@timestamp": "2023-01-22T09:05:05.000Z", + "@timestamp": "2024-01-22T09:05:05.000Z", "destination": { "geo": { "continent_name": "Europe", @@ -1129,7 +1129,7 @@ ] }, { - "@timestamp": "2023-01-22T10:52:34.000Z", + "@timestamp": "2024-01-22T10:52:34.000Z", "destination": { "ip": "ff02:0000:0000:0000:0000:0000:0000:0016", "mac": "90-10-12-34-56-78" @@ -1185,7 +1185,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:05.000Z", + "@timestamp": "2024-01-05T20:17:05.000Z", "destination": { "geo": { "city_name": "London", @@ -1265,7 +1265,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -1341,7 +1341,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -1424,7 +1424,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -1499,7 +1499,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -1629,7 +1629,7 @@ ] }, { - "@timestamp": "2023-06-28T04:35:30.000Z", + "@timestamp": "2024-06-28T04:35:30.000Z", "destination": { "ip": "10.251.1.1", "mac": "0A-EA-10-00-F0-06", @@ -1682,7 +1682,7 @@ ] }, { - "@timestamp": "2023-06-28T04:30:32.000Z", + "@timestamp": "2024-06-28T04:30:32.000Z", "destination": { "ip": "10.251.1.1", "mac": "0A-EA-10-00-F0-06" diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-ubiquiti.log-expected.json b/packages/iptables/data_stream/log/_dev/test/pipeline/test-ubiquiti.log-expected.json index f452eb79dd6..334b49b0616 100644 --- a/packages/iptables/data_stream/log/_dev/test/pipeline/test-ubiquiti.log-expected.json +++ b/packages/iptables/data_stream/log/_dev/test/pipeline/test-ubiquiti.log-expected.json @@ -1,7 +1,7 @@ { "expected": [ { - "@timestamp": "2023-01-05T20:17:05.000Z", + "@timestamp": "2024-01-05T20:17:05.000Z", "destination": { "geo": { "city_name": "London", @@ -81,7 +81,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -157,7 +157,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -240,7 +240,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -315,7 +315,7 @@ ] }, { - "@timestamp": "2023-01-05T20:17:01.000Z", + "@timestamp": "2024-01-05T20:17:01.000Z", "destination": { "ip": "192.168.2.25", "mac": "90-10-20-76-8D-20", @@ -390,7 +390,7 @@ ] }, { - "@timestamp": "2023-05-05T20:46:45.000Z", + "@timestamp": "2024-05-05T20:46:45.000Z", "destination": { "port": 443 }, @@ -432,7 +432,7 @@ ] }, { - "@timestamp": "2023-05-05T20:46:46.000Z", + "@timestamp": "2024-05-05T20:46:46.000Z", "destination": { "port": 7914 }, @@ -474,7 +474,7 @@ ] }, { - "@timestamp": "2023-05-05T20:46:46.000Z", + "@timestamp": "2024-05-05T20:46:46.000Z", "destination": { "port": 51179 }, @@ -515,7 +515,7 @@ ] }, { - "@timestamp": "2023-05-05T20:47:09.000Z", + "@timestamp": "2024-05-05T20:47:09.000Z", "destination": { "port": 443 }, @@ -557,7 +557,7 @@ ] }, { - "@timestamp": "2023-05-05T20:46:56.000Z", + "@timestamp": "2024-05-05T20:46:56.000Z", "destination": { "port": 51182 }, @@ -598,7 +598,7 @@ ] }, { - "@timestamp": "2023-05-05T20:45:44.000Z", + "@timestamp": "2024-05-05T20:45:44.000Z", "destination": { "port": 49209 }, diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 8879095e7d9..8b79f3ac547 100644 --- a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -385,6 +385,6 @@ on_failure: - append: field: error.message value: >- - Processor '{{ _ingest.on_failure_processor_type }}' in pipeline {{{_ingest.pipeline}}} - {{#_ingest.on_failure_processor_tag}} with tag '{{ _ingest.on_failure_processor_tag }}'{{/_ingest.on_failure_processor_tag}} - failed with message '{{ _ingest.on_failure_message }}' \ No newline at end of file + Processor '{{{ _ingest.on_failure_processor_type }}}' in pipeline {{{_ingest.pipeline}}} + {{{#_ingest.on_failure_processor_tag}}} with tag '{{{ _ingest.on_failure_processor_tag }}}'{{{/_ingest.on_failure_processor_tag}}} + failed with message '{{{ _ingest.on_failure_message }}}' \ No newline at end of file diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml index e6613d9433a..94b3360f3e1 100644 --- a/packages/iptables/manifest.yml +++ b/packages/iptables/manifest.yml @@ -1,6 +1,6 @@ name: iptables title: Iptables -version: "1.16.1" +version: "1.16.2" description: Collect logs from Iptables with Elastic Agent. type: integration icons: diff --git a/packages/juniper_srx/changelog.yml b/packages/juniper_srx/changelog.yml index 816dc77b196..6e7427efb29 100644 --- a/packages/juniper_srx/changelog.yml +++ b/packages/juniper_srx/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.21.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.21.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml index 700ac039240..b61cc62baf3 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/atp.yml @@ -362,7 +362,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml index b633ea41a92..87120d07abb 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -377,7 +377,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml index a11dfa9ffcb..687a110a760 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/flow.yml @@ -361,7 +361,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml index 51637f35cf5..4e30cb73c0a 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/idp.yml @@ -286,7 +286,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml index d54b4037d1d..7806225773e 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/ids.yml @@ -362,7 +362,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml index b9dbbeabda8..3e614baa769 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/secintel.yml @@ -348,7 +348,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error \ No newline at end of file diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml index 940dab42c27..2cc409cc542 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/system.yml @@ -30,7 +30,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # Cleanup process and tag fields. # If juniper.srx.process and juniper.srx.tag fields are not already set, or has `-`, then set them using newly parsed grok pattern in system_structured_brief @@ -53,7 +53,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # 2. if: negotiation.message ~ /^success.*/ - dissect: if: "ctx._temp_?.negotiation?.message != null && ctx._temp_.negotiation.message.startsWith('success')" @@ -63,7 +63,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.negotiation target_field: juniper.srx.negotiation @@ -79,7 +79,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.fw target_field: juniper.srx.firewall @@ -98,7 +98,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.rtslib_dfwsm target_field: juniper.srx.rtslib_dfwsm @@ -113,7 +113,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.ip_mon_reth_scan target_field: juniper.srx.ip_mon_reth_scan @@ -133,7 +133,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - convert: field: _temp_.dpdk.port_number type: integer @@ -183,7 +183,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.rtlog_conn_error target_field: juniper.srx.rtlog_conn_error @@ -245,7 +245,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.ping_test target_field: juniper.srx.ping_test @@ -261,7 +261,7 @@ processors: on_failure: - append: field: error.message - value: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + value: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - rename: field: _temp_.kern_arp_addr_change target_field: juniper.srx.kern_arp_addr_change @@ -820,7 +820,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml index f9452f64d47..071e7a35f3e 100644 --- a/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml +++ b/packages/juniper_srx/data_stream/log/elasticsearch/ingest_pipeline/utm.yml @@ -389,7 +389,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/juniper_srx/manifest.yml b/packages/juniper_srx/manifest.yml index 64c2fe3b761..9f1952149f8 100644 --- a/packages/juniper_srx/manifest.yml +++ b/packages/juniper_srx/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: juniper_srx title: Juniper SRX -version: "1.21.1" +version: "1.21.2" description: Collect logs from Juniper SRX devices with Elastic Agent. categories: ["network", "security", "firewall_security"] type: integration diff --git a/packages/modsecurity/changelog.yml b/packages/modsecurity/changelog.yml index 8bc64ac1d76..69e597579d4 100644 --- a/packages/modsecurity/changelog.yml +++ b/packages/modsecurity/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.18.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.18.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml index cf9c545650f..cc3a823c6e2 100644 --- a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/apache-modsec.yml @@ -69,7 +69,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # rename ecs - rename: @@ -95,11 +95,11 @@ processors: - set: field: _temps.url if: ctx.json.transaction.local_port == 443 - value: "https://{{json.request.headers.Host}}:{{json.transaction.#local_port}}{{url.original}}" + value: "https://{{{json.request.headers.Host}}}:{{json.transaction.#local_port}}{{{url.original}}}" - set: field: _temps.url if: ctx.json.transaction.local_port == 80 - value: "http://{{json.request.headers.Host}}:{{json.transaction.#local_port}}{{url.original}}" + value: "http://{{{json.request.headers.Host}}}:{{json.transaction.#local_port}}{{{url.original}}}" - uri_parts: field: _temps.url ignore_failure: true diff --git a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml index 05d694cf256..bc04c3d227e 100644 --- a/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml +++ b/packages/modsecurity/data_stream/auditlog/elasticsearch/ingest_pipeline/nginx-modsec.yml @@ -59,7 +59,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # rename ecs - rename: @@ -86,11 +86,11 @@ processors: - set: field: _temps.url if: ctx.json.transaction.host_port == 443 - value: "https://{{json.transaction.request.headers.Host}}:{{json.transaction.host_port}}{{json.transaction.request.uri}}" + value: "https://{{{json.transaction.request.headers.Host}}}:{{{json.transaction.host_port}}}{{{json.transaction.request.uri}}}" - set: field: _temps.url if: ctx.json.transaction.host_port == 80 - value: "http://{{json.transaction.request.headers.Host}}:{{json.transaction.host_port}}{{json.transaction.request.uri}}" + value: "http://{{{json.transaction.request.headers.Host}}}:{{{json.transaction.host_port}}}{{{json.transaction.request.uri}}}" - uri_parts: field: _temps.url ignore_failure: true diff --git a/packages/modsecurity/manifest.yml b/packages/modsecurity/manifest.yml index 206410ee5af..6051735ed62 100644 --- a/packages/modsecurity/manifest.yml +++ b/packages/modsecurity/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: modsecurity title: "ModSecurity Audit" -version: "1.18.1" +version: "1.18.2" description: Collect logs from ModSecurity with Elastic Agent type: integration categories: diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index 76855fe85c4..f662762fb32 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "1.20.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml index afd4477b368..2f3e7f3b2f7 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -193,7 +193,7 @@ processors: if: "ctx.user?.name != null" - set: field: network.direction - value: "{{network.direction}}bound" + value: "{{{network.direction}}}bound" if: ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/ - remove: field: diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 5a75792946b..229b9a58fe2 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense -version: "1.20.1" +version: "1.20.2" description: Collect logs from pfSense and OPNsense with Elastic Agent. type: integration icons: diff --git a/packages/sophos/changelog.yml b/packages/sophos/changelog.yml index 7017f42abd9..ce39b7e3ebe 100644 --- a/packages/sophos/changelog.yml +++ b/packages/sophos/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.9.2" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "3.9.1" changes: - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. diff --git a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml index 3dfee3f1a0c..0bac9906212 100644 --- a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml @@ -38,7 +38,7 @@ processors: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error @@ -134,7 +134,7 @@ processors: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error @@ -229,7 +229,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml index 3eebc9467f6..2c798438b43 100644 --- a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml +++ b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml @@ -160,7 +160,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml index bd286a6d7d0..b84bacf16ce 100644 --- a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml +++ b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml @@ -88,7 +88,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml index 32ce078b468..36869be50bf 100644 --- a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml +++ b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml @@ -310,7 +310,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml index df95ecdc894..5bfee4d9659 100644 --- a/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml +++ b/packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml @@ -265,7 +265,7 @@ on_failure: - append: field: error.message value: |- - Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}" + Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}" - set: field: event.kind value: pipeline_error diff --git a/packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml b/packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml index a252eb59ddc..7191fb42192 100644 --- a/packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml +++ b/packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/default.yml @@ -86,7 +86,7 @@ processors: ignore_missing: true - set: field: _temp_.time - value: "{{sophos.xg.date}} {{sophos.xg.time}}" + value: "{{{sophos.xg.date}}} {{{sophos.xg.time}}}" if: ctx.sophos?.xg?.date != null && ctx.sophos?.xg?.time != null - set: field: _temp_.time @@ -140,7 +140,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" - date: tag: "date_utc_fallback" timezone: UTC @@ -156,7 +156,7 @@ processors: field: error.message value: "fail-{{{ _ingest.on_failure_processor_tag }}}" - fail: - message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}" + message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}" # Sets starts, end and duration when start and duration is known - script: diff --git a/packages/sophos/manifest.yml b/packages/sophos/manifest.yml index f0918175f5d..64959f40815 100644 --- a/packages/sophos/manifest.yml +++ b/packages/sophos/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: sophos title: Sophos -version: "3.9.1" +version: "3.9.2" description: Collect logs from Sophos with Elastic Agent. categories: - "security" diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index ea294350fa8..6b7eb4bcb45 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.24.4" + changes: + - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines. + type: bugfix + link: https://github.com/elastic/integrations/pull/11315 - version: "2.24.3" changes: - description: Add conditional check to grok processors in known_certs pipeline to prevent failures. diff --git a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml index f0efef2daf7..e7e7e2a3288 100644 --- a/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml +++ b/packages/zeek/data_stream/socks/elasticsearch/ingest_pipeline/third-party.yml @@ -37,6 +37,6 @@ on_failure: field: error.message value: >- error in third party api pipeline: - error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} - with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} - {{ _ingest.on_failure_message }} + error in [{{{_ingest.on_failure_processor_type}}}] processor{{{#_ingest.on_failure_processor_tag}}} + with tag [{{{_ingest.on_failure_processor_tag }}}]{{{/_ingest.on_failure_processor_tag}}} + {{{ _ingest.on_failure_message }}} diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index f2c866dce5a..d33eb868a3a 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: "2.24.3" +version: "2.24.4" description: Collect logs from Zeek with Elastic Agent. type: integration icons: