From a7726a98513694175e2887d128b6305209894e39 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 17 Sep 2024 06:41:05 +0930 Subject: [PATCH] cloudflare_logpush: retain firewall event zone names (#11132) --- packages/cloudflare_logpush/changelog.yml | 5 + .../pipeline/test-pipeline-firewall-event.log | 3 +- ...-pipeline-firewall-event.log-expected.json | 102 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 4 + .../firewall_event/fields/fields.yml | 6 ++ packages/cloudflare_logpush/docs/README.md | 1 + packages/cloudflare_logpush/manifest.yml | 2 +- 7 files changed, 121 insertions(+), 2 deletions(-) diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml index b10dba5e32f..4cd6fa0ac62 100644 --- a/packages/cloudflare_logpush/changelog.yml +++ b/packages/cloudflare_logpush/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.25.0" + changes: + - description: Retain zone name for firewall events. + type: enhancement + link: https://github.com/elastic/integrations/pull/11132 - version: "1.24.0" changes: - description: Support new JA4 fields from HTTP Requests logs. diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log index 7d5344e029f..0a7ba42a0a7 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log @@ -1,3 +1,4 @@ {"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"2022-05-31T05:23:43Z","ClientRequestHost":"xyz.example.com","ClientASN":15169} {"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623","ClientRequestHost":"xyz.example.com","ClientASN":15169} -{"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623000000000","ClientRequestHost":"xyz.example.com","ClientASN":15169} \ No newline at end of file +{"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623000000000","ClientRequestHost":"xyz.example.com","ClientASN":15169} +{"EdgeEndTimestamp":"2024-09-11T12:57:10Z","EdgeResponseBytes":7062,"EdgeResponseStatus":200,"EdgeStartTimestamp":"2024-09-11T12:57:10Z","ContentScanObjResults":[],"ContentScanObjSizes":[],"ContentScanObjTypes":[],"Cookies":{},"LeakedCredentialCheckResult":"none","ParentRayID":"00","RayID":"abcdef1234567890","RequestHeaders":{},"ResponseHeaders":{},"SmartRouteColoID":0,"UpperTierColoID":0,"ZoneName":"nota.real.name","ClientASN":12345,"ClientCountry":"ch","ClientDeviceType":"desktop","ClientIP":"192.168.1.1","ClientIPClass":"noRecord","ClientMTLSAuthCertFingerprint":"","ClientMTLSAuthStatus":"unknown","ClientRegionCode":"ZH","ClientRequestBytes":9942,"ClientRequestHost":"logs.nota.real.name","ClientRequestMethod":"GET","ClientRequestPath":"/foo/bar","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"https://logs.nota.real.name","ClientRequestScheme":"https","ClientRequestSource":"eyeball","ClientRequestURI":"/foo/bar","ClientRequestUserAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":56450,"ClientTCPRTTMs":6,"ClientXRequestedWith":"","SecurityAction":"","SecurityActions":[],"SecurityRuleDescription":"","SecurityRuleID":"","SecurityRuleIDs":[],"SecuritySources":[],"OriginResponseDurationMs":0,"OriginResponseStatus":0,"OriginResponseTime":0} diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json index 5e49b554055..d29295e945b 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json +++ b/packages/cloudflare_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json @@ -401,6 +401,108 @@ }, "version": "2.1" } + }, + { + "cloudflare_logpush": { + "firewall_event": { + "client": { + "asn": { + "value": 12345 + }, + "country": "ch", + "ip": "192.168.1.1", + "ip_class": "noRecord", + "request": { + "host": "logs.nota.real.name", + "method": "GET", + "path": "/foo/bar", + "protocol": "HTTP/2", + "scheme": "https", + "user": { + "agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36" + } + } + }, + "edge": { + "response": { + "status": 200 + } + }, + "origin": { + "response": { + "status": 0 + } + }, + "ray": { + "id": "abcdef1234567890" + }, + "zone": { + "name": "nota.real.name" + } + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"EdgeEndTimestamp\":\"2024-09-11T12:57:10Z\",\"EdgeResponseBytes\":7062,\"EdgeResponseStatus\":200,\"EdgeStartTimestamp\":\"2024-09-11T12:57:10Z\",\"ContentScanObjResults\":[],\"ContentScanObjSizes\":[],\"ContentScanObjTypes\":[],\"Cookies\":{},\"LeakedCredentialCheckResult\":\"none\",\"ParentRayID\":\"00\",\"RayID\":\"abcdef1234567890\",\"RequestHeaders\":{},\"ResponseHeaders\":{},\"SmartRouteColoID\":0,\"UpperTierColoID\":0,\"ZoneName\":\"nota.real.name\",\"ClientASN\":12345,\"ClientCountry\":\"ch\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"192.168.1.1\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRegionCode\":\"ZH\",\"ClientRequestBytes\":9942,\"ClientRequestHost\":\"logs.nota.real.name\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/foo/bar\",\"ClientRequestProtocol\":\"HTTP/2\",\"ClientRequestReferer\":\"https://logs.nota.real.name\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"eyeball\",\"ClientRequestURI\":\"/foo/bar\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36\",\"ClientSSLCipher\":\"AEAD-AES128-GCM-SHA256\",\"ClientSSLProtocol\":\"TLSv1.3\",\"ClientSrcPort\":56450,\"ClientTCPRTTMs\":6,\"ClientXRequestedWith\":\"\",\"SecurityAction\":\"\",\"SecurityActions\":[],\"SecurityRuleDescription\":\"\",\"SecurityRuleID\":\"\",\"SecurityRuleIDs\":[],\"SecuritySources\":[],\"OriginResponseDurationMs\":0,\"OriginResponseStatus\":0,\"OriginResponseTime\":0}", + "type": [ + "info" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + }, + "version": "2" + }, + "network": { + "protocol": "http" + }, + "related": { + "hosts": [ + "logs.nota.real.name" + ], + "ip": [ + "192.168.1.1" + ] + }, + "source": { + "as": { + "number": 12345 + }, + "geo": { + "country_iso_code": "ch" + }, + "ip": "192.168.1.1" + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "url": { + "domain": "logs.nota.real.name", + "path": "/foo/bar", + "scheme": "https" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Chrome", + "original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + "os": { + "name": "Linux" + }, + "version": "99.0.4844.51" + } } ] } \ No newline at end of file diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml index 73624c380a9..ef7ba6371f8 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml @@ -265,6 +265,10 @@ processors: field: json.Source target_field: cloudflare_logpush.firewall_event.source ignore_missing: true + - rename: + field: json.ZoneName + target_field: cloudflare_logpush.firewall_event.zone.name + ignore_missing: true - append: field: related.ip value: '{{{source.ip}}}' diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml index eb9b2fccb02..7ea1cdd8aa1 100644 --- a/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml +++ b/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml @@ -124,6 +124,12 @@ - name: timestamp type: date description: The date and time the event occurred at the edge. + - name: zone + type: group + fields: + - name: name + type: keyword + description: The human-readable name of the zone. - name: log.source.address type: keyword description: Source address from which the log event was read / sent from. diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md index 8c86fffd843..9470b3d3904 100644 --- a/packages/cloudflare_logpush/docs/README.md +++ b/packages/cloudflare_logpush/docs/README.md @@ -1295,6 +1295,7 @@ An example event for `firewall_event` looks as following: | cloudflare_logpush.firewall_event.rule.id | The Cloudflare security product-specific RuleID triggered by this request. | keyword | | cloudflare_logpush.firewall_event.source | The Cloudflare security product triggered by this request. | keyword | | cloudflare_logpush.firewall_event.timestamp | The date and time the event occurred at the edge. | date | +| cloudflare_logpush.firewall_event.zone.name | The human-readable name of the zone. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml index 8c70ceb5f4c..806fae59ecc 100644 --- a/packages/cloudflare_logpush/manifest.yml +++ b/packages/cloudflare_logpush/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cloudflare_logpush title: Cloudflare Logpush -version: "1.24.0" +version: "1.25.0" description: Collect and parse logs from Cloudflare API with Elastic Agent. type: integration categories: