From 7991db95637147ac713610c67caa21654fb6cbec Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 24 Sep 2024 15:05:30 -0400 Subject: [PATCH] [pfsense] Add SNORT log processing (#11182) * [pfsense] Add SNORT log processing * Update changelog with PR number * Address code review * Address code review feedback --- packages/pfsense/changelog.yml | 5 + .../_dev/test/pipeline/test-pfsense-snort.log | 2 + .../test-pfsense-snort.log-expected.json | 186 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 5 +- .../elasticsearch/ingest_pipeline/snort.yml | 14 ++ .../pfsense/data_stream/log/fields/fields.yml | 24 +++ packages/pfsense/docs/README.md | 7 + packages/pfsense/manifest.yml | 2 +- 8 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log create mode 100644 packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log-expected.json create mode 100644 packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml diff --git a/packages/pfsense/changelog.yml b/packages/pfsense/changelog.yml index 715ebd4814e..c91bd7f1082 100644 --- a/packages/pfsense/changelog.yml +++ b/packages/pfsense/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.20.0" + changes: + - description: Add SNORT log processing + type: enhancement + link: https://github.com/elastic/integrations/pull/11182 - version: "1.19.2" changes: - description: Fix firewall ICMPv6 message parsing error diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log new file mode 100644 index 00000000000..0657932e22b --- /dev/null +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log @@ -0,0 +1,2 @@ +<190>Jul 23 18:12:00 snort[87537]: [136:1:1] (spp_reputation) packets blacklisted [Classification: Potentially Bad Traffic] [Priority: 2] {TCP} 67.43.156.1:16856 -> 89.160.20.128:2222 +<190>Jul 23 18:12:00 snort[87537]: [119:4:1] (http_inspect) BARE BYTE UNICODE ENCODING [Classification: Not Suspicious Traffic] [Priority: 3] {TCP} 67.43.156.0:63651 -> 89.160.20.128:8080 diff --git a/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log-expected.json b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log-expected.json new file mode 100644 index 00000000000..45789fdb972 --- /dev/null +++ b/packages/pfsense/data_stream/log/_dev/test/pipeline/test-pfsense-snort.log-expected.json @@ -0,0 +1,186 @@ +{ + "expected": [ + { + "@timestamp": "2024-07-23T18:12:00.000-04:00", + "destination": { + "address": "89.160.20.128", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.128", + "port": 2222 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "<190>Jul 23 18:12:00 snort[87537]: [136:1:1] (spp_reputation) packets blacklisted [Classification: Potentially Bad Traffic] [Priority: 2] {TCP} 67.43.156.1:16856 -> 89.160.20.128:2222", + "provider": "snort", + "timezone": "-04:00" + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "[136:1:1] (spp_reputation) packets blacklisted [Classification: Potentially Bad Traffic] [Priority: 2] {TCP} 67.43.156.1:16856 -> 89.160.20.128:2222", + "network": { + "protocol": "tcp", + "type": "ipv4" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "snort", + "pid": 87537 + }, + "related": { + "ip": [ + "89.160.20.128", + "67.43.156.1" + ] + }, + "snort": { + "alert_message": "packets blacklisted", + "classification": "Potentially Bad Traffic", + "generator_id": "136", + "preprocessor": "spp_reputation", + "priority": 2, + "signature_id": "1", + "signature_revision": "1" + }, + "source": { + "address": "67.43.156.1", + "as": { + "number": 35908 + }, + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + } + }, + "ip": "67.43.156.1", + "port": 16856 + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-07-23T18:12:00.000-04:00", + "destination": { + "address": "89.160.20.128", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.128", + "port": 8080 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "<190>Jul 23 18:12:00 snort[87537]: [119:4:1] (http_inspect) BARE BYTE UNICODE ENCODING [Classification: Not Suspicious Traffic] [Priority: 3] {TCP} 67.43.156.0:63651 -> 89.160.20.128:8080", + "provider": "snort", + "timezone": "-04:00" + }, + "log": { + "syslog": { + "priority": 190 + } + }, + "message": "[119:4:1] (http_inspect) BARE BYTE UNICODE ENCODING [Classification: Not Suspicious Traffic] [Priority: 3] {TCP} 67.43.156.0:63651 -> 89.160.20.128:8080", + "network": { + "protocol": "tcp", + "type": "ipv4" + }, + "observer": { + "type": "firewall", + "vendor": "netgate" + }, + "process": { + "name": "snort", + "pid": 87537 + }, + "related": { + "ip": [ + "89.160.20.128", + "67.43.156.0" + ] + }, + "snort": { + "alert_message": "BARE BYTE UNICODE ENCODING", + "classification": "Not Suspicious Traffic", + "generator_id": "119", + "preprocessor": "http_inspect", + "priority": 3, + "signature_id": "4", + "signature_revision": "1" + }, + "source": { + "address": "67.43.156.0", + "as": { + "number": 35908 + }, + "geo": { + "continent_name": "Asia", + "country_iso_code": "BT", + "country_name": "Bhutan", + "location": { + "lat": 27.5, + "lon": 90.5 + } + }, + "ip": "67.43.156.0", + "port": 63651 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml index dc08f7c5d8c..de94cc13bc3 100644 --- a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -86,8 +86,11 @@ processors: - pipeline: name: '{{ IngestPipeline "squid" }}' if: ctx.event.provider == 'squid' + - pipeline: + name: '{{ IngestPipeline "snort" }}' + if: ctx.event.provider == 'snort' - drop: - if: '!["filterlog", "openvpn", "charon", "dhcpd", "dhclient", "dhcp6c", "unbound", "haproxy", "php-fpm", "squid"].contains(ctx.event?.provider)' + if: '!["filterlog", "openvpn", "charon", "dhcpd", "dhclient", "dhcp6c", "unbound", "haproxy", "php-fpm", "squid", "snort"].contains(ctx.event?.provider)' - append: field: event.category value: network diff --git a/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml new file mode 100644 index 00000000000..17281c0f978 --- /dev/null +++ b/packages/pfsense/data_stream/log/elasticsearch/ingest_pipeline/snort.yml @@ -0,0 +1,14 @@ +--- +description: Pipeline for PFsense SNORT logs. +processors: + - grok: + field: message + patterns: + - '\[%{NUMBER:snort.generator_id}:%{NUMBER:snort.signature_id}:%{NUMBER:snort.signature_revision}\] \(%{DATA:snort.preprocessor}\) %{GREEDYDATA:snort.alert_message} \[Classification: %{DATA:snort.classification}\] \[Priority: %{NONNEGINT:snort.priority:long}\] \{%{WORD:network.protocol}\} %{IP:source.address}:%{NUMBER:source.port:long} -> %{IP:destination.address}:%{NUMBER:destination.port:long}' + - lowercase: + field: network.protocol + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{{ _ingest.on_failure_message }}}" diff --git a/packages/pfsense/data_stream/log/fields/fields.yml b/packages/pfsense/data_stream/log/fields/fields.yml index 10b0ecdc8c6..a74806ce387 100644 --- a/packages/pfsense/data_stream/log/fields/fields.yml +++ b/packages/pfsense/data_stream/log/fields/fields.yml @@ -281,3 +281,27 @@ - name: hierarchy_status type: keyword description: The proxy hierarchy route; the route Content Gateway used to retrieve the object. +- name: snort + type: group + fields: + - name: alert_message + type: keyword + description: Snort alert message. + - name: classification + type: keyword + description: Snort classification. + - name: generator_id + type: keyword + description: Snort generator id. + - name: preprocessor + type: keyword + description: Snort preprocessor. + - name: priority + type: long + description: Snort priority. + - name: signature_id + type: keyword + description: Snort signature id. + - name: signature_revision + type: keyword + description: Snort signature revision. \ No newline at end of file diff --git a/packages/pfsense/docs/README.md b/packages/pfsense/docs/README.md index ec7df736386..497f5c8391a 100644 --- a/packages/pfsense/docs/README.md +++ b/packages/pfsense/docs/README.md @@ -384,6 +384,13 @@ An example event for `log` looks as following: | server.ip | IP address of the server (IPv4 or IPv6). | ip | | server.mac | MAC address of the server. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | server.port | Port of the server. | long | +| snort.alert_message | Snort alert message. | keyword | +| snort.classification | Snort classification. | keyword | +| snort.generator_id | Snort generator id. | keyword | +| snort.preprocessor | Snort preprocessor. | keyword | +| snort.priority | Snort priority. | long | +| snort.signature_id | Snort signature id. | keyword | +| snort.signature_revision | Snort signature revision. | keyword | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | | source.as.organization.name | Organization name. | keyword | diff --git a/packages/pfsense/manifest.yml b/packages/pfsense/manifest.yml index 1ad425ad6ce..32e544ce313 100644 --- a/packages/pfsense/manifest.yml +++ b/packages/pfsense/manifest.yml @@ -1,6 +1,6 @@ name: pfsense title: pfSense -version: "1.19.2" +version: "1.20.0" description: Collect logs from pfSense and OPNsense with Elastic Agent. type: integration icons: