cloudflare_logpush: retain firewall event zone names (#11132)

elastic · Sep 16, 2024 · a7726a9 · a7726a9
1 parent cae44e9
commit a7726a9
Show file tree

Hide file tree

Showing 7 changed files with 121 additions and 2 deletions.
diff --git a/packages/cloudflare_logpush/changelog.yml b/packages/cloudflare_logpush/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.25.0"
+  changes:
+    - description: Retain zone name for firewall events.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11132
 - version: "1.24.0"
   changes:
     - description: Support new JA4 fields from HTTP Requests logs.

diff --git a/...re_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log b/...re_logpush/data_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log
@@ -1,3 +1,4 @@
 {"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"2022-05-31T05:23:43Z","ClientRequestHost":"xyz.example.com","ClientASN":15169}
 {"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623","ClientRequestHost":"xyz.example.com","ClientASN":15169}
-{"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623000000000","ClientRequestHost":"xyz.example.com","ClientASN":15169}
+{"ClientRequestScheme":"https","MatchIndex":1,"ClientRefererHost":"abc.example.com","Source":"firewallrules","ClientRequestUserAgent":"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","ClientRefererPath":"/abc/checkout","Metadata":{"filter":"1ced07e066a34abf8b14f2a99593bc8d","type":"customer"},"EdgeResponseStatus":403,"ClientRequestProtocol":"HTTP/1.1","OriginatorRayID":"00","RayID":"713d477539b55c29","ClientRequestMethod":"GET","ClientIP":"175.16.199.0","ClientRequestPath":"/abc/checkout","Action":"block","Kind":"firewall","RuleID":"7dc666e026974dab84884c73b3e2afe1","ClientIPClass":"searchEngine","ClientASNDescription":"CLOUDFLARENET","ClientCountry":"us","ClientRefererQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","ClientRequestQuery":"?sourcerer=(default%3A(id%3A!n%2CselectedPatterns%3A!(eqldemo%2C%27logs-endpoint.*-eqldemo%27%2C%27logs-system.*-eqldemo%27%2C%27logs-windows.*-eqldemo%27%2Cmetricseqldemo)))&timerange=(global%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.199Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.200Z%27%2CtoStr%3Anow))%2Ctimeline%3A(linkTo%3A!()%2Ctimerange%3A(from%3A%272022-04-05T00%3A00%3A01.201Z%27%2CfromStr%3Anow-24h%2Ckind%3Arelative%2Cto%3A%272022-04-06T00%3A00%3A01.202Z%27%2CtoStr%3Anow)))","OriginResponseStatus":0,"EdgeColoCode":"IAD","ClientRefererScheme":"referer URL scheme","Datetime":"1653974623000000000","ClientRequestHost":"xyz.example.com","ClientASN":15169}
+{"EdgeEndTimestamp":"2024-09-11T12:57:10Z","EdgeResponseBytes":7062,"EdgeResponseStatus":200,"EdgeStartTimestamp":"2024-09-11T12:57:10Z","ContentScanObjResults":[],"ContentScanObjSizes":[],"ContentScanObjTypes":[],"Cookies":{},"LeakedCredentialCheckResult":"none","ParentRayID":"00","RayID":"abcdef1234567890","RequestHeaders":{},"ResponseHeaders":{},"SmartRouteColoID":0,"UpperTierColoID":0,"ZoneName":"nota.real.name","ClientASN":12345,"ClientCountry":"ch","ClientDeviceType":"desktop","ClientIP":"192.168.1.1","ClientIPClass":"noRecord","ClientMTLSAuthCertFingerprint":"","ClientMTLSAuthStatus":"unknown","ClientRegionCode":"ZH","ClientRequestBytes":9942,"ClientRequestHost":"logs.nota.real.name","ClientRequestMethod":"GET","ClientRequestPath":"/foo/bar","ClientRequestProtocol":"HTTP/2","ClientRequestReferer":"https://logs.nota.real.name","ClientRequestScheme":"https","ClientRequestSource":"eyeball","ClientRequestURI":"/foo/bar","ClientRequestUserAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36","ClientSSLCipher":"AEAD-AES128-GCM-SHA256","ClientSSLProtocol":"TLSv1.3","ClientSrcPort":56450,"ClientTCPRTTMs":6,"ClientXRequestedWith":"","SecurityAction":"","SecurityActions":[],"SecurityRuleDescription":"","SecurityRuleID":"","SecurityRuleIDs":[],"SecuritySources":[],"OriginResponseDurationMs":0,"OriginResponseStatus":0,"OriginResponseTime":0}
diff --git a/...a_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json b/...a_stream/firewall_event/_dev/test/pipeline/test-pipeline-firewall-event.log-expected.json
@@ -401,6 +401,108 @@
                 },
                 "version": "2.1"
             }
+        },
+        {
+            "cloudflare_logpush": {
+                "firewall_event": {
+                    "client": {
+                        "asn": {
+                            "value": 12345
+                        },
+                        "country": "ch",
+                        "ip": "192.168.1.1",
+                        "ip_class": "noRecord",
+                        "request": {
+                            "host": "logs.nota.real.name",
+                            "method": "GET",
+                            "path": "/foo/bar",
+                            "protocol": "HTTP/2",
+                            "scheme": "https",
+                            "user": {
+                                "agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36"
+                            }
+                        }
+                    },
+                    "edge": {
+                        "response": {
+                            "status": 200
+                        }
+                    },
+                    "origin": {
+                        "response": {
+                            "status": 0
+                        }
+                    },
+                    "ray": {
+                        "id": "abcdef1234567890"
+                    },
+                    "zone": {
+                        "name": "nota.real.name"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "kind": "event",
+                "original": "{\"EdgeEndTimestamp\":\"2024-09-11T12:57:10Z\",\"EdgeResponseBytes\":7062,\"EdgeResponseStatus\":200,\"EdgeStartTimestamp\":\"2024-09-11T12:57:10Z\",\"ContentScanObjResults\":[],\"ContentScanObjSizes\":[],\"ContentScanObjTypes\":[],\"Cookies\":{},\"LeakedCredentialCheckResult\":\"none\",\"ParentRayID\":\"00\",\"RayID\":\"abcdef1234567890\",\"RequestHeaders\":{},\"ResponseHeaders\":{},\"SmartRouteColoID\":0,\"UpperTierColoID\":0,\"ZoneName\":\"nota.real.name\",\"ClientASN\":12345,\"ClientCountry\":\"ch\",\"ClientDeviceType\":\"desktop\",\"ClientIP\":\"192.168.1.1\",\"ClientIPClass\":\"noRecord\",\"ClientMTLSAuthCertFingerprint\":\"\",\"ClientMTLSAuthStatus\":\"unknown\",\"ClientRegionCode\":\"ZH\",\"ClientRequestBytes\":9942,\"ClientRequestHost\":\"logs.nota.real.name\",\"ClientRequestMethod\":\"GET\",\"ClientRequestPath\":\"/foo/bar\",\"ClientRequestProtocol\":\"HTTP/2\",\"ClientRequestReferer\":\"https://logs.nota.real.name\",\"ClientRequestScheme\":\"https\",\"ClientRequestSource\":\"eyeball\",\"ClientRequestURI\":\"/foo/bar\",\"ClientRequestUserAgent\":\"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36\",\"ClientSSLCipher\":\"AEAD-AES128-GCM-SHA256\",\"ClientSSLProtocol\":\"TLSv1.3\",\"ClientSrcPort\":56450,\"ClientTCPRTTMs\":6,\"ClientXRequestedWith\":\"\",\"SecurityAction\":\"\",\"SecurityActions\":[],\"SecurityRuleDescription\":\"\",\"SecurityRuleID\":\"\",\"SecurityRuleIDs\":[],\"SecuritySources\":[],\"OriginResponseDurationMs\":0,\"OriginResponseStatus\":0,\"OriginResponseTime\":0}",
+                "type": [
+                    "info"
+                ]
+            },
+            "http": {
+                "request": {
+                    "method": "GET"
+                },
+                "response": {
+                    "status_code": 200
+                },
+                "version": "2"
+            },
+            "network": {
+                "protocol": "http"
+            },
+            "related": {
+                "hosts": [
+                    "logs.nota.real.name"
+                ],
+                "ip": [
+                    "192.168.1.1"
+                ]
+            },
+            "source": {
+                "as": {
+                    "number": 12345
+                },
+                "geo": {
+                    "country_iso_code": "ch"
+                },
+                "ip": "192.168.1.1"
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ],
+            "url": {
+                "domain": "logs.nota.real.name",
+                "path": "/foo/bar",
+                "scheme": "https"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Chrome",
+                "original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
+                "os": {
+                    "name": "Linux"
+                },
+                "version": "99.0.4844.51"
+            }
         }
     ]
 }
diff --git a/...s/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml b/...s/cloudflare_logpush/data_stream/firewall_event/elasticsearch/ingest_pipeline/default.yml
@@ -265,6 +265,10 @@ processors:
       field: json.Source
       target_field: cloudflare_logpush.firewall_event.source
       ignore_missing: true
+  - rename:
+      field: json.ZoneName
+      target_field: cloudflare_logpush.firewall_event.zone.name
+      ignore_missing: true
   - append:
       field: related.ip
       value: '{{{source.ip}}}'

diff --git a/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml b/packages/cloudflare_logpush/data_stream/firewall_event/fields/fields.yml
@@ -124,6 +124,12 @@
     - name: timestamp
       type: date
       description: The date and time the event occurred at the edge.
+    - name: zone
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: The human-readable name of the zone.
 - name: log.source.address
   type: keyword
   description: Source address from which the log event was read / sent from.
diff --git a/packages/cloudflare_logpush/docs/README.md b/packages/cloudflare_logpush/docs/README.md
@@ -1295,6 +1295,7 @@ An example event for `firewall_event` looks as following:
 | cloudflare_logpush.firewall_event.rule.id | The Cloudflare security product-specific RuleID triggered by this request. | keyword |
 | cloudflare_logpush.firewall_event.source | The Cloudflare security product triggered by this request. | keyword |
 | cloudflare_logpush.firewall_event.timestamp | The date and time the event occurred at the edge. | date |
+| cloudflare_logpush.firewall_event.zone.name | The human-readable name of the zone. | keyword |
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |

diff --git a/packages/cloudflare_logpush/manifest.yml b/packages/cloudflare_logpush/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: cloudflare_logpush
 title: Cloudflare Logpush
-version: "1.24.0"
+version: "1.25.0"
 description: Collect and parse logs from Cloudflare API with Elastic Agent.
 type: integration
 categories: