sdd_all: use triple-brace templating in complex templates (#11315)

This is a second pass at replacing double brace template marks with triple brace marks. It addresses more complex uses that the sed rewrite did not find by using PCRE via perl. [git-generate] for f in $( ( for p in $( yq 'select(.owner.github == "elastic/sec-deployment-and-devices")|.name' packages/**/manifest.yml \ | grep -v -- '---' ); do rg -l -g '*.yml' "[^{]\{\{[^{][ .a-zA-Z0-9_]*[^}]}}[^}]" packages/$p done )|grep "elasticsearch/ingest_pipeline"|sort|uniq ); do perl -pi -e 's/(?<!\{)(\{\{[^{][ .a-zA-Z0-9_]*[^}]}})(?!\})/{$1}/g' $f done for p in $(git diff --name-only HEAD~1|cut -d/ -f1,2|sort|uniq); do ( cd $p elastic-package test pipeline -g elastic-package changelog add \ --description "Use triple-brace Mustache templating when referencing variables in ingest pipelines." \ --type bugfix \ --next patch \ --link #11315 )>/dev/null 2>&1 done
elastic · Oct 8, 2024 · 2621bb0 · 2621bb0
1 parent 26029a5
commit 2621bb0
Show file tree

Hide file tree

Showing 56 changed files with 169 additions and 104 deletions.
diff --git a/packages/cef/changelog.yml b/packages/cef/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.17.4"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "2.17.3"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/cp-pipeline.yml
@@ -379,7 +379,7 @@ on_failure:
   - append:
       field: error.message
       value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
   - set:
       field: event.kind
       value: pipeline_error
diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -197,7 +197,7 @@ on_failure:
   - append:
       field: error.message
       value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
   - set:
       field: event.kind
       value: pipeline_error
diff --git a/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml b/packages/cef/data_stream/log/elasticsearch/ingest_pipeline/fp-pipeline.yml
@@ -25,7 +25,7 @@ on_failure:
   - append:
       field: error.message
       value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
   - set:
       field: event.kind
       value: pipeline_error
diff --git a/packages/cef/manifest.yml b/packages/cef/manifest.yml
@@ -1,6 +1,6 @@
 name: cef
 title: Common Event Format (CEF)
-version: "2.17.3"
+version: "2.17.4"
 description: Collect logs from CEF Logs with Elastic Agent.
 categories:
   - security

diff --git a/packages/checkpoint/changelog.yml b/packages/checkpoint/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.34.2"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "1.34.1"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

diff --git a/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml b/packages/checkpoint/data_stream/firewall/elasticsearch/ingest_pipeline/default.yml
@@ -28,7 +28,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   - kv:
       tag: "kv_syslog_structured_semicolon_colon"
       field: syslog5424_sd
@@ -52,7 +52,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   - grok:
       field: syslog5424_sd
       if: ctx.checkpoint == null
@@ -86,7 +86,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   - foreach:
       field: checkpoint
       ignore_missing: true
@@ -160,7 +160,7 @@ processors:
                   field: error.message
                   value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
               - fail:
-                  message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+                  message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   # Some log events lack loguid and time, so to avoid potential
   # collisions hash the complete line in those rare cases.
   - fingerprint:
@@ -1066,7 +1066,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   - rename:
       field: checkpoint._temp_unixms
       target_field: "@timestamp"

diff --git a/packages/checkpoint/manifest.yml b/packages/checkpoint/manifest.yml
@@ -1,6 +1,6 @@
 name: checkpoint
 title: Check Point
-version: "1.34.1"
+version: "1.34.2"
 description: Collect logs from Check Point with Elastic Agent.
 type: integration
 format_version: "3.0.3"

diff --git a/packages/cisco_ftd/changelog.yml b/packages/cisco_ftd/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.4.3"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "3.4.2"
   changes:
     - description: "Fix grok failure with username with spaces on ftd messageID."

diff --git a/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_ftd/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -1815,7 +1815,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
   - grok:
       field: "destination.user.name"
       tag: "grok_destination_user_name"
@@ -1833,7 +1833,7 @@ processors:
             field: error.message
             value: "fail-{{{ _ingest.on_failure_processor_tag }}}"
         - fail:
-            message: "Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message: {{ _ingest.on_failure_message }}"
+            message: "Processor {{{ _ingest.on_failure_processor_type }}} with tag {{{ _ingest.on_failure_processor_tag }}} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message: {{{ _ingest.on_failure_message }}}"
 
   #
   # Normalize protocol names

diff --git a/packages/cisco_ftd/manifest.yml b/packages/cisco_ftd/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ftd
 title: Cisco FTD
-version: "3.4.2"
+version: "3.4.3"
 description: Collect logs from Cisco FTD with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/cisco_ios/changelog.yml b/packages/cisco_ios/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.27.2"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "1.27.1"
   changes:
     - description: Handle timestamp starting with the year such as 'yyyy MMM d HH:mm:ss.SSS z'

diff --git a/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/cisco_ios/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -461,7 +461,7 @@ on_failure:
   - append:
       field: error.message
       value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
   - set:
       field: event.kind
       value: pipeline_error
diff --git a/packages/cisco_ios/manifest.yml b/packages/cisco_ios/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ios
 title: Cisco IOS
-version: "1.27.1"
+version: "1.27.2"
 description: Collect logs from Cisco IOS with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/cisco_ise/changelog.yml b/packages/cisco_ise/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.2"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "1.23.1"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

diff --git a/...sco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml b/...sco_ise/data_stream/log/elasticsearch/ingest_pipeline/pipeline_passed_authentications.yml
@@ -575,6 +575,6 @@ on_failure:
   - append:
       field: error.message
       value: >-
-        Processor '{{ _ingest.on_failure_processor_type }}'
-        {{#_ingest.on_failure_processor_tag}}with tag '{{ _ingest.on_failure_processor_tag }}'
-        {{/_ingest.on_failure_processor_tag}}failed with message '{{ _ingest.on_failure_message }}'
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}'
diff --git a/packages/cisco_ise/manifest.yml b/packages/cisco_ise/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_ise
 title: Cisco ISE
-version: "1.23.1"
+version: "1.23.2"
 description: Collect logs from Cisco ISE with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/cisco_secure_email_gateway/changelog.yml b/packages/cisco_secure_email_gateway/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.24.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "1.24.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

diff --git a/...ail_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml b/...ail_gateway/data_stream/log/elasticsearch/ingest_pipeline/pipeline_consolidated_event.yml
@@ -534,4 +534,4 @@ on_failure:
   - append:
       field: error.message
       value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
diff --git a/packages/cisco_secure_email_gateway/manifest.yml b/packages/cisco_secure_email_gateway/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: cisco_secure_email_gateway
 title: Cisco Secure Email Gateway
-version: "1.24.0"
+version: "1.24.1"
 description: Collect logs from Cisco Secure Email Gateway with Elastic Agent.
 type: integration
 categories:

diff --git a/packages/goflow2/changelog.yml b/packages/goflow2/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.1.1"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "0.1.0"
   changes:
     - description: Initial version of the package

diff --git a/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml b/packages/goflow2/data_stream/sflow/elasticsearch/ingest_pipeline/default.yml
@@ -271,7 +271,7 @@ processors:
 on_failure:
 - set:
     field: error.message
-    value: Processor {{ _ingest.on_failure_processor_type }} with tag {{ _ingest.on_failure_processor_tag
-      }} in pipeline {{ _ingest.on_failure_pipeline }} failed with message {{ _ingest.on_failure_message
-      }} - Source {{_source}}
+    value: Processor {{{ _ingest.on_failure_processor_type }}} with tag {{ _ingest.on_failure_processor_tag
+      }} in pipeline {{{ _ingest.on_failure_pipeline }}} failed with message {{ _ingest.on_failure_message
+      }} - Source {{{_source}}}
     tag: set_error_message_on_failure
diff --git a/packages/goflow2/manifest.yml b/packages/goflow2/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: goflow2
 title: "GoFlow2 logs"
-version: 0.1.0
+version: 0.1.1
 description: "Collect logs from goflow2 with Elastic Agent."
 type: integration
 categories:

diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.2"
+  changes:
+    - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11315
 - version: "1.16.1"
   changes:
     - description: Invoke community_id processor only for supported protocols