Skip to content

Commit

Permalink
slp_all: use triple-brace templating in templates (#11317)
Browse files Browse the repository at this point in the history
The mustache templating system used by ingest pipelines has two levels of
escaping available, not escaped (triple stache) and HTML escaped
(double stache) — see man mustache[1] under "tag types: variables". This can
lead to data corruption, particularly in cases where an operating system has
chosen to use a character requiring escaping in its path syntax.

[1]http://mustache.github.io/mustache.5.html

[git-generate]
for f in $(
	(
		for p in $(
			yq 'select(.owner.github == "elastic/sec-linux-platform")|.name' packages/**/manifest.yml \
			| grep -v -- '---'
		); do
			rg -l -g '*.yml' "[^{]\{\{[^{][ .a-zA-Z0-9_]*[^}]}}[^}]" packages/$p
		done
	)|grep "elasticsearch/ingest_pipeline"|sort|uniq
); do
	perl -pi -e 's/(?<!\{)(\{\{[^{][ .a-zA-Z0-9_]*[^}]}})(?!\})/{$1}/g' $f
done
for p in $(git diff --name-only HEAD~1|cut -d/ -f1,2|sort|uniq); do
	(
		cd $p
		elastic-package test pipeline -g
		elastic-package changelog add \
			--description "Use triple-brace Mustache templating when referencing variables in ingest pipelines." \
			--type bugfix \
			--next patch \
			--link #11317
	)>/dev/null 2>&1
done
  • Loading branch information
efd6 authored Oct 8, 2024
1 parent 2621bb0 commit 5febbdd
Show file tree
Hide file tree
Showing 33 changed files with 66 additions and 41 deletions.
5 changes: 5 additions & 0 deletions packages/auditd/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "3.20.2"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
type: bugfix
link: https://github.com/elastic/integrations/pull/11317
- version: "3.20.1"
changes:
- description: "Preserve auditd.log.record_type and fallback to auditd.log.SYSCALL"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1992,8 +1992,8 @@ processors:
ignore_missing: true
processor:
set:
field: "{{_ingest._value.target}}"
value: "{{_ingest._value.value}}"
field: "{{{_ingest._value.target}}}"
value: "{{{_ingest._value.value}}}"
- set:
if: "ctx.auditd.log?.record_type == 'SYSTEM_BOOT' || ctx.auditd.log?.record_type == 'SYSTEM_SHUTDOWN'"
field: event.category
Expand Down Expand Up @@ -2037,12 +2037,12 @@ processors:
- set:
if: "ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID'"
field: container.name
value: "{{ auditd.log.vm }}"
value: "{{{ auditd.log.vm }}}"
ignore_empty_value: true
- set:
if: "ctx.auditd.log?.record_type == 'VIRT_MACHINE_ID'"
field: container.runtime
value: "{{ auditd.log.virt }}"
value: "{{{ auditd.log.virt }}}"
ignore_empty_value: true
- set:
if: >
Expand Down
2 changes: 1 addition & 1 deletion packages/auditd/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: auditd
title: Auditd Logs
version: "3.20.1"
version: "3.20.2"
description: Collect logs from Linux audit daemon with Elastic Agent.
type: integration
icons:
Expand Down
5 changes: 5 additions & 0 deletions packages/cloud_defend/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.2.6"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
type: bugfix
link: https://github.com/elastic/integrations/pull/11317
- version: "1.2.5"
changes:
- description: Update integration support matrix
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ processors:
value: 'cloud-defend'
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
value: '{{{_ingest.timestamp}}}'
- set:
field: event.dataset
value: 'cloud_defend.alerts'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ processors:
value: 'cloud_defend.file'
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
value: '{{{_ingest.timestamp}}}'
- set:
field: event.module
value: 'cloud_defend'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ processors:
value: 'cloud_defend.process'
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
value: '{{{_ingest.timestamp}}}'
- set:
field: event.module
value: 'cloud_defend'
Expand Down
2 changes: 1 addition & 1 deletion packages/cloud_defend/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: 3.0.0
name: cloud_defend
title: "Defend for Containers"
version: 1.2.5
version: 1.2.6
source:
license: "Elastic-2.0"
description: "Elastic Defend for Containers (BETA) provides cloud-native runtime protections for containerized environments."
Expand Down
5 changes: 5 additions & 0 deletions packages/network_traffic/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.31.2"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
type: bugfix
link: https://github.com/elastic/integrations/pull/11317
- version: "1.31.1"
changes:
- description: Add `event.module` to datastreams
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,11 @@ processors:

- append:
field: related.hash
value: "{{tls.server.ja3s}}"
value: "{{{tls.server.ja3s}}}"
if: "ctx?.tls?.server?.ja3s != null"
- append:
field: related.hash
value: "{{tls.client.ja3}}"
value: "{{{tls.client.ja3}}}"
if: "ctx?.tls?.client?.ja3 != null"
allow_duplicates: false

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ on_failure:
- append:
field: error.message
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
- set:
field: event.kind
value: pipeline_error
2 changes: 1 addition & 1 deletion packages/network_traffic/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: "3.0.0"
name: network_traffic
title: Network Packet Capture
version: "1.31.1"
version: "1.31.2"
description: Capture and analyze network traffic from a host with Elastic Agent.
type: integration
categories:
Expand Down
5 changes: 5 additions & 0 deletions packages/sysmon_linux/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.7.1"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
type: bugfix
link: https://github.com/elastic/integrations/pull/11317
- version: "1.7.0"
changes:
- description: Tighten IPv4 extraction from IPv4-mapped IPv6 addresses.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ processors:
value: event
- set:
field: event.code
value: '{{winlog.event_id}}'
value: '{{{winlog.event_id}}}'
- set:
field: event.action
value: "log"
Expand Down Expand Up @@ -878,7 +878,7 @@ processors:
ignore_missing: true
- append:
field: related.hosts
value: "{{dns.question.name}}"
value: "{{{dns.question.name}}}"
allow_duplicates: false
if: ctx?.dns?.question?.name != null && ctx?.dns?.question?.name != ""
- remove:
Expand All @@ -892,7 +892,7 @@ processors:
processor:
append:
field: related.ip
value: "{{_ingest._value}}"
value: "{{{_ingest._value}}}"
allow_duplicates: false
ignore_failure: true
- community_id:
Expand All @@ -913,13 +913,13 @@ processors:
if: ctx?.winlog?.event_data?.User != null
- set:
field: user.domain
value: "{{_temp.user_parts.0}}"
value: "{{{_temp.user_parts.0}}}"
ignore_failure: true
ignore_empty_value: true
if: ctx?._temp?.user_parts != null && ctx._temp.user_parts.size() == 2
- set:
field: user.name
value: "{{_temp.user_parts.1}}"
value: "{{{_temp.user_parts.1}}}"
ignore_failure: true
ignore_empty_value: true
if: ctx?._temp?.user_parts != null && ctx._temp.user_parts.size() == 2
Expand Down Expand Up @@ -1178,19 +1178,19 @@ processors:

- append:
field: related.user
value: "{{user.name}}"
value: "{{{user.name}}}"
ignore_failure: true
allow_duplicates: false
if: ctx?.user?.name != null && ctx.user.name != ""
- append:
field: related.ip
value: "{{source.ip}}"
value: "{{{source.ip}}}"
ignore_failure: true
allow_duplicates: false
if: ctx?.source?.ip != null && ctx.source.ip != ""
- append:
field: related.ip
value: "{{destination.ip}}"
value: "{{{destination.ip}}}"
ignore_failure: true
allow_duplicates: false
if: ctx?.destination?.ip != null && ctx.destination.ip != ""
Expand Down Expand Up @@ -1368,4 +1368,4 @@ on_failure:
- append:
field: "error.message"
value: |-
Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
2 changes: 1 addition & 1 deletion packages/sysmon_linux/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: sysmon_linux
title: Sysmon for Linux
version: "1.7.0"
version: "1.7.1"
description: Collect Sysmon Linux logs with Elastic Agent.
type: integration
categories:
Expand Down
5 changes: 5 additions & 0 deletions packages/system_audit/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.10.3"
changes:
- description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.
type: bugfix
link: https://github.com/elastic/integrations/pull/11317
- version: "1.10.2"
changes:
- description: capture root requirement
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@ processors:
on_failure:
- set:
field: error.message
value: "{{ _ingest.on_failure_message }}"
value: "{{{ _ingest.on_failure_message }}}"
- set:
field: package.architecture
copy_from: system.audit.package.arch
if: ctx.package?.architecture == "" && ctx.system?.audit?.package?.arch != ""
on_failure:
- set:
field: error.message
value: "{{ _ingest.on_failure_message }}"
value: "{{{ _ingest.on_failure_message }}}"

###########
# Cleanup #
Expand Down
2 changes: 1 addition & 1 deletion packages/system_audit/manifest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: system_audit
title: System Audit
description: Collect various logs & metrics from System Audit modules with Elastic Agent.
type: integration
version: "1.10.2"
version: "1.10.3"
conditions:
kibana:
version: '^8.7.1'
Expand Down

0 comments on commit 5febbdd

Please sign in to comment.