From 8107f32598cd1692f6cf915a6d267fe2f7a0c797 Mon Sep 17 00:00:00 2001 From: Nicholas Berlin <56366649+nicholasberlin@users.noreply.github.com> Date: Tue, 30 Jul 2024 13:10:09 -0400 Subject: [PATCH] Copy log.syslog.appname to process.name (#10646) * Copy log.syslog.appname to process.name Before the syslog processor was added to the default filebeat yaml configuration, process.name was set by a Grok ingest pipeline. However, when the syslog processor was added a condition was also added to the Grok ingest pipeline, `if: ctx.log?.syslog == null`. This condition always evaluates to false when the data is supplied by the syslog processor, and therefore `process.name` is not extraced from the syslog message. This commit copies the process name from `log.syslog.appname` to `process.name` before an pipeline processors which use it are executed. With this change, `process.name`, `event.category`, and `event.type` are now populated when data arrives from the syslog processor. --- .github/CODEOWNERS | 2 +- packages/system/changelog.yml | 5 + .../pipeline/test-host-syslog-processor.json | 144 +++++++++++++ ...t-host-syslog-processor.json-expected.json | 199 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 6 + packages/system/manifest.yml | 2 +- 6 files changed, 356 insertions(+), 2 deletions(-) create mode 100644 packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json create mode 100644 packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json-expected.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 61243a00df9..fd7680766ef 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -328,7 +328,7 @@ /packages/sysmon_linux @elastic/sec-linux-platform /packages/system @elastic/obs-infraobs-integrations /packages/system/changelog.yml @elastic/obs-infraobs-integrations @elastic/sec-linux-platform @elastic/sec-windows-platform -/packages/system/data_stream/auth @elastic/sec-windows-platform +/packages/system/data_stream/auth @elastic/sec-linux-platform /packages/system/data_stream/security @elastic/sec-windows-platform /packages/system/data_stream/application @elastic/sec-windows-platform /packages/system/data_stream/core @elastic/obs-infraobs-integrations diff --git a/packages/system/changelog.yml b/packages/system/changelog.yml index ffe830be123..a52e893f903 100644 --- a/packages/system/changelog.yml +++ b/packages/system/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.60.1" + changes: + - description: Ensure process.name is populated from syslog messages + type: bugfix + link: https://github.com/elastic/integrations/pull/10646 - version: "1.60.0" changes: - description: Add caseless fields to process events. diff --git a/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json b/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json new file mode 100644 index 00000000000..cafcff6bd9f --- /dev/null +++ b/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json @@ -0,0 +1,144 @@ +{ + "events": [ + { + "event" : { + "original": "<165>1 2022-01-11T22:14:15.003Z mymachine.example.com eventslog 1024 ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"][examplePriority@32473 class=\"high\"] this is the message" + }, + "@timestamp": "2022-01-11T22:14:15.003Z", + "log": { + "syslog": { + "priority": 165, + "facility": { + "code": 20, + "name": "local4" + }, + "severity": { + "code": 5, + "name": "Notice" + }, + "hostname": "mymachine.example.com", + "appname": "eventslog", + "procid": "1024", + "msgid": "ID47", + "version": "1", + "structured_data": { + "exampleSDID@32473": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + }, + "examplePriority@32473": { + "class": "high" + } + } + } + }, + "message": "this is the message" + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "event" : { + "original": "<165>1 2022-01-11T22:14:15.003Z mymachine.example.com groupadd 1024 ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"][examplePriority@32473 class=\"high\"] this is the message" + }, + "log": { + "syslog": { + "priority": 165, + "facility": { + "code": 20, + "name": "local4" + }, + "severity": { + "code": 5, + "name": "Notice" + }, + "hostname": "mymachine.example.com", + "appname": "groupdel", + "procid": "1024", + "msgid": "ID47", + "version": "1", + "structured_data": { + "exampleSDID@32473": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + }, + "examplePriority@32473": { + "class": "high" + } + } + } + }, + "message": "this is the message" + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "event" : { + "original": "<165>1 2022-01-11T22:14:15.003Z mymachine.example.com useradd 1024 ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"][examplePriority@32473 class=\"high\"] this is the message" + }, + "log": { + "syslog": { + "priority": 165, + "facility": { + "code": 20, + "name": "local4" + }, + "severity": { + "code": 5, + "name": "Notice" + }, + "hostname": "mymachine.example.com", + "appname": "useradd", + "procid": "1024", + "msgid": "ID47", + "version": "1", + "structured_data": { + "exampleSDID@32473": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + }, + "examplePriority@32473": { + "class": "high" + } + } + } + }, + "message": "this is the message" + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "event" : { + "original": "<165>1 2022-01-11T22:14:15.003Z mymachine.example.com usermod 1024 ID47 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"][examplePriority@32473 class=\"high\"] this is the message" + }, + "log": { + "syslog": { + "priority": 165, + "facility": { + "code": 20, + "name": "local4" + }, + "severity": { + "code": 5, + "name": "Notice" + }, + "hostname": "mymachine.example.com", + "appname": "usermod", + "procid": "1024", + "msgid": "ID47", + "version": "1", + "structured_data": { + "exampleSDID@32473": { + "iut": "3", + "eventSource": "Application", + "eventID": "1011" + }, + "examplePriority@32473": { + "class": "high" + } + } + } + }, + "message": "this is the message" + } + ] +} diff --git a/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json-expected.json b/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json-expected.json new file mode 100644 index 00000000000..a261896b33b --- /dev/null +++ b/packages/system/data_stream/auth/_dev/test/pipeline/test-host-syslog-processor.json-expected.json @@ -0,0 +1,199 @@ +{ + "expected": [ + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event" + }, + "log": { + "syslog": { + "priority": 30, + "appname": "eventslog", + "facility": { + "code": 20, + "name": "local4" + }, + "hostname": "mymachine.example.com", + "msgid": "ID47", + "priority": 165, + "procid": "1024", + "severity": { + "code": 5, + "name": "Notice" + }, + "structured_data": { + "examplePriority@32473": { + "class": "high" + }, + "exampleSDID@32473": { + "eventID": "1011", + "eventSource": "Application", + "iut": "3" + } + }, + "version": "1" + } + }, + "message": "this is the message", + "process": { + "name": "eventslog" + } + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "category": [ + "iam" + ], + "kind": "event", + "outcome": "success", + "type": [ + "group", + "deletion" + ] + }, + "log": { + "syslog": { + "priority": 30, + "appname": "groupdel", + "facility": { + "code": 20, + "name": "local4" + }, + "hostname": "mymachine.example.com", + "msgid": "ID47", + "priority": 165, + "procid": "1024", + "severity": { + "code": 5, + "name": "Notice" + }, + "structured_data": { + "examplePriority@32473": { + "class": "high" + }, + "exampleSDID@32473": { + "eventID": "1011", + "eventSource": "Application", + "iut": "3" + } + }, + "version": "1" + } + }, + "message": "this is the message", + "process": { + "name": "groupdel" + } + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "category": [ + "iam" + ], + "kind": "event", + "outcome": "success", + "type": [ + "user", + "creation" + ] + }, + "log": { + "syslog": { + "priority": 30, + "appname": "useradd", + "facility": { + "code": 20, + "name": "local4" + }, + "hostname": "mymachine.example.com", + "msgid": "ID47", + "priority": 165, + "procid": "1024", + "severity": { + "code": 5, + "name": "Notice" + }, + "structured_data": { + "examplePriority@32473": { + "class": "high" + }, + "exampleSDID@32473": { + "eventID": "1011", + "eventSource": "Application", + "iut": "3" + } + }, + "version": "1" + } + }, + "message": "this is the message", + "process": { + "name": "useradd" + } + }, + { + "@timestamp": "2022-01-11T22:14:15.003Z", + "ecs": { + "version": "8.11.0" + }, + "event": { + "kind": "event", + "category": [ + "iam" + ], + "kind": "event", + "outcome": "success", + "type": [ + "user", + "change" + ] + }, + "log": { + "syslog": { + "priority": 30, + "appname": "usermod", + "facility": { + "code": 20, + "name": "local4" + }, + "hostname": "mymachine.example.com", + "msgid": "ID47", + "priority": 165, + "procid": "1024", + "severity": { + "code": 5, + "name": "Notice" + }, + "structured_data": { + "examplePriority@32473": { + "class": "high" + }, + "exampleSDID@32473": { + "eventID": "1011", + "eventSource": "Application", + "iut": "3" + } + }, + "version": "1" + } + }, + "message": "this is the message", + "process": { + "name": "usermod" + } + } + ] +} diff --git a/packages/system/data_stream/auth/elasticsearch/ingest_pipeline/default.yml b/packages/system/data_stream/auth/elasticsearch/ingest_pipeline/default.yml index 001765c3c10..c321ccce5d3 100644 --- a/packages/system/data_stream/auth/elasticsearch/ingest_pipeline/default.yml +++ b/packages/system/data_stream/auth/elasticsearch/ingest_pipeline/default.yml @@ -303,6 +303,12 @@ processors: ctx.event.action = "ssh_login"; ctx.event.outcome = "failure"; } + - set: + tag: set_process_name-log_syslog_appname + field: process.name + copy_from: log.syslog.appname + override: false + ignore_empty_value: true - append: tag: append_category-iam field: event.category diff --git a/packages/system/manifest.yml b/packages/system/manifest.yml index 013cae27fa7..625b35cfc3b 100644 --- a/packages/system/manifest.yml +++ b/packages/system/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.2 name: system title: System -version: "1.60.0" +version: "1.60.1" description: Collect system logs and metrics from your servers with Elastic Agent. type: integration categories: