From ef0bcefbffde014b6dab4db259c44d78dd145200 Mon Sep 17 00:00:00 2001 From: Brijesh Khunt <123942796+brijesh-elastic@users.noreply.github.com> Date: Wed, 7 Aug 2024 18:38:49 +0530 Subject: [PATCH 01/13] [watchguard_firebox] Add date format pattern to parse syslog timestamp (#10708) - Add additional date format pattern to handle whitespace padding --- .../docker/sample_logs/watchguard_firebox.log | 1 + packages/watchguard_firebox/changelog.yml | 5 ++ .../log/_dev/test/pipeline/test-alarm.log | 1 + .../pipeline/test-alarm.log-expected.json | 71 +++++++++++++++++++ .../log/_dev/test/system/test-udp-config.yml | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 1 + .../data_stream/log/sample_event.json | 12 ++-- packages/watchguard_firebox/docs/README.md | 12 ++-- packages/watchguard_firebox/manifest.yml | 2 +- 9 files changed, 93 insertions(+), 14 deletions(-) diff --git a/packages/watchguard_firebox/_dev/deploy/docker/sample_logs/watchguard_firebox.log b/packages/watchguard_firebox/_dev/deploy/docker/sample_logs/watchguard_firebox.log index 89b4954e060..caaf72a7ef6 100644 --- a/packages/watchguard_firebox/_dev/deploy/docker/sample_logs/watchguard_firebox.log +++ b/packages/watchguard_firebox/_dev/deploy/docker/sample_logs/watchguard_firebox.log @@ -26,3 +26,4 @@ <142>May 20 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-01-10T11:49:17) firewall: msg_id="3000-0175" Deny Trusted External tcp 10.0.1.2 175.16.199.1 37930 80 msg="ProxyDrop: HTTP Virus found" proxy_act="HTTP-Client.Standard.1" md5="69630e4574ec6798239b091cda43dca0" virus="EICAR-Test-File (not a virus)" host="175.16.199.1" path="/eicar.com.txt" (HTTP proxy-00) <142>May 20 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-01-10T11:49:17) firewall: msg_id="3000-0176" Allow Trusted External tcp 10.0.1.2 175.16.199.1 37932 80 msg="HTTP request" proxy_act="HTTP-Client.Standard.1" op="GET" dstname="175.16.199.1" arg="/index.html" sent_bytes="176" rcvd_bytes="517" elapsed_time="0.002265 sec(s)" (HTTP-proxy-00) <142>May 20 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-01-10T11:49:17) firewall: msg_id="1AFF-0001" Deny 1-Trusted 6-Ext-access tcp 10.0.1.2 192.168.53.82 60654 80 msg="ProxyDeny: HTTP server response timeout" (HTTP-proxy-00) +<142>Aug 6 08:51:27 WatchGuard-Firebox FVE6035FD3AE3 (2024-08-06T03:31:27) firewall[10]: msg_id="3000-0172" Blocked port: Traffic detected from 2a02:cf40:: to 175.16.199.1 on port 513. (Blocked Ports) diff --git a/packages/watchguard_firebox/changelog.yml b/packages/watchguard_firebox/changelog.yml index a6643d05088..f6393b5b5f9 100644 --- a/packages/watchguard_firebox/changelog.yml +++ b/packages/watchguard_firebox/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.1.1" + changes: + - description: Add date format pattern to parse syslog timestamp. + type: bugfix + link: https://github.com/elastic/integrations/pull/10708 - version: "0.1.0" changes: - description: Initial release. diff --git a/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log b/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log index 534e5890146..cbe4f31b2ab 100644 --- a/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log +++ b/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log @@ -20,3 +20,4 @@ <142>May 10 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-05-10T09:49:05) firewall[10]: msg_id="3000-0170" The total number of current sessions (1024) has reached the high water mark (1024). (HTTP-Client.Standard.1-px) <142>May 10 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-05-10T09:49:05) firewall[10]: msg_id="3000-0171" The number of connections (2048) has reached the configured limit (2048). (HTTP-Client.Standard.1-px) <142>May 10 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-05-10T09:49:05) firewall[10]: msg_id="3000-0172" Blocked port: Traffic detected from 10.0.1.2 to 175.16.199.0 on port 513. (Blocked Ports) +<142>Aug 6 08:51:27 WatchGuard-Firebox FVE6035FD3AE3 (2024-08-06T03:31:27) firewall[10]: msg_id="3000-0172" Blocked port: Traffic detected from 2a02:cf40:: to 175.16.199.1 on port 513. (Blocked Ports) diff --git a/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log-expected.json b/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log-expected.json index 5d36653538d..8aa845c3432 100644 --- a/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log-expected.json +++ b/packages/watchguard_firebox/data_stream/log/_dev/test/pipeline/test-alarm.log-expected.json @@ -1498,6 +1498,77 @@ "timestamp": "2024-05-10T09:49:05.000Z" } } + }, + { + "@timestamp": "2024-08-06T08:51:27.000+05:30", + "destination": { + "ip": "175.16.199.1" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "alert", + "original": "<142>Aug 6 08:51:27 WatchGuard-Firebox FVE6035FD3AE3 (2024-08-06T03:31:27) firewall[10]: msg_id=\"3000-0172\" Blocked port: Traffic detected from 2a02:cf40:: to 175.16.199.1 on port 513. (Blocked Ports)", + "outcome": "unknown", + "timezone": "+05:30", + "type": [ + "denied" + ] + }, + "log": { + "syslog": { + "appname": "firewall", + "hostname": "WatchGuard-Firebox", + "priority": 142, + "procid": "10" + } + }, + "message": "Blocked port: Traffic detected from 2a02:cf40:: to 175.16.199.1 on port 513.", + "observer": { + "hostname": "WatchGuard-Firebox", + "product": "Firebox", + "serial_number": "FVE6035FD3AE3", + "type": "firewall", + "vendor": "WatchGuard" + }, + "related": { + "hosts": [ + "WatchGuard-Firebox" + ], + "ip": [ + "175.16.199.1", + "2a02:cf40::" + ] + }, + "rule": { + "name": [ + "Blocked Ports" + ] + }, + "source": { + "ip": "2a02:cf40::" + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ], + "watchguard_firebox": { + "log": { + "destination_ip": "175.16.199.1", + "log_type": "alarm", + "msg_id": "3000-0172", + "policy_name": "Blocked Ports", + "port": 513, + "serial_number": "FVE6035FD3AE3", + "source_ip": "2a02:cf40::", + "syslog_timestamp": "2024-08-06T08:51:27.000+05:30", + "timestamp": "2024-08-06T03:31:27.000Z" + } + } } ] } \ No newline at end of file diff --git a/packages/watchguard_firebox/data_stream/log/_dev/test/system/test-udp-config.yml b/packages/watchguard_firebox/data_stream/log/_dev/test/system/test-udp-config.yml index 098f324dcec..37983bdba81 100644 --- a/packages/watchguard_firebox/data_stream/log/_dev/test/system/test-udp-config.yml +++ b/packages/watchguard_firebox/data_stream/log/_dev/test/system/test-udp-config.yml @@ -8,4 +8,4 @@ data_stream: preserve_original_event: true preserve_duplicate_custom_fields: true assert: - hit_count: 28 + hit_count: 29 diff --git a/packages/watchguard_firebox/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/watchguard_firebox/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 1af0541b28d..27d8145b04d 100644 --- a/packages/watchguard_firebox/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/watchguard_firebox/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -71,6 +71,7 @@ processors: formats: - MMM d HH:mm:ss - MMM dd HH:mm:ss + - MMM d HH:mm:ss on_failure: - append: field: error.message diff --git a/packages/watchguard_firebox/data_stream/log/sample_event.json b/packages/watchguard_firebox/data_stream/log/sample_event.json index e6825f5016d..747ada66203 100644 --- a/packages/watchguard_firebox/data_stream/log/sample_event.json +++ b/packages/watchguard_firebox/data_stream/log/sample_event.json @@ -1,15 +1,15 @@ { "@timestamp": "2024-01-16T15:19:05.000Z", "agent": { - "ephemeral_id": "d064e318-a25b-4194-981a-ac29f9146cbb", - "id": "0a23b9c0-d1c4-47ce-b003-9b5041aff6e9", + "ephemeral_id": "5a9738b6-025a-4df4-861e-1cc1eea5c622", + "id": "7aaba523-565c-4597-bc42-59135436336b", "name": "docker-fleet-agent", "type": "filebeat", "version": "8.13.0" }, "data_stream": { "dataset": "watchguard_firebox.log", - "namespace": "69604", + "namespace": "76887", "type": "logs" }, "destination": { @@ -37,7 +37,7 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "0a23b9c0-d1c4-47ce-b003-9b5041aff6e9", + "id": "7aaba523-565c-4597-bc42-59135436336b", "snapshot": false, "version": "8.13.0" }, @@ -57,7 +57,7 @@ "email" ], "dataset": "watchguard_firebox.log", - "ingested": "2024-07-10T10:21:38Z", + "ingested": "2024-08-07T05:26:04Z", "kind": "event", "original": "<139>Jan 16 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-01-19T08:48:15) firewall: msg_id=\"1BFF-000F\" Allow 1-Trusted 0-External tcp 10.0.1.2 175.16.199.1 39398 25 msg=\"SMTP request\" proxy_act=\"SMTP-Outgoing.1\" rcvd_bytes=\"272\" sent_bytes=\"282\" sender=\"tester@testnet.com\" recipients=\"wg@localhost\" server_ssl=\"ECDHE-RSA-AES256-GCMSHA384\" client_ssl=\"AES128-SHA256\" tls_profile=\"TLS-Client.Standard\" (SMTP-proxy-00)", "outcome": "success", @@ -71,7 +71,7 @@ }, "log": { "source": { - "address": "192.168.241.7:39831" + "address": "192.168.240.4:51247" }, "syslog": { "appname": "firewall", diff --git a/packages/watchguard_firebox/docs/README.md b/packages/watchguard_firebox/docs/README.md index 78cfac6b599..8f59b347aec 100644 --- a/packages/watchguard_firebox/docs/README.md +++ b/packages/watchguard_firebox/docs/README.md @@ -61,15 +61,15 @@ An example event for `log` looks as following: { "@timestamp": "2024-01-16T15:19:05.000Z", "agent": { - "ephemeral_id": "d064e318-a25b-4194-981a-ac29f9146cbb", - "id": "0a23b9c0-d1c4-47ce-b003-9b5041aff6e9", + "ephemeral_id": "5a9738b6-025a-4df4-861e-1cc1eea5c622", + "id": "7aaba523-565c-4597-bc42-59135436336b", "name": "docker-fleet-agent", "type": "filebeat", "version": "8.13.0" }, "data_stream": { "dataset": "watchguard_firebox.log", - "namespace": "69604", + "namespace": "76887", "type": "logs" }, "destination": { @@ -97,7 +97,7 @@ An example event for `log` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "0a23b9c0-d1c4-47ce-b003-9b5041aff6e9", + "id": "7aaba523-565c-4597-bc42-59135436336b", "snapshot": false, "version": "8.13.0" }, @@ -117,7 +117,7 @@ An example event for `log` looks as following: "email" ], "dataset": "watchguard_firebox.log", - "ingested": "2024-07-10T10:21:38Z", + "ingested": "2024-08-07T05:26:04Z", "kind": "event", "original": "<139>Jan 16 15:19:05 WatchGuard-Firebox FVE6035FD3AE3 (2024-01-19T08:48:15) firewall: msg_id=\"1BFF-000F\" Allow 1-Trusted 0-External tcp 10.0.1.2 175.16.199.1 39398 25 msg=\"SMTP request\" proxy_act=\"SMTP-Outgoing.1\" rcvd_bytes=\"272\" sent_bytes=\"282\" sender=\"tester@testnet.com\" recipients=\"wg@localhost\" server_ssl=\"ECDHE-RSA-AES256-GCMSHA384\" client_ssl=\"AES128-SHA256\" tls_profile=\"TLS-Client.Standard\" (SMTP-proxy-00)", "outcome": "success", @@ -131,7 +131,7 @@ An example event for `log` looks as following: }, "log": { "source": { - "address": "192.168.241.7:39831" + "address": "192.168.240.4:51247" }, "syslog": { "appname": "firewall", diff --git a/packages/watchguard_firebox/manifest.yml b/packages/watchguard_firebox/manifest.yml index 562f4299448..83fbafb5cbf 100644 --- a/packages/watchguard_firebox/manifest.yml +++ b/packages/watchguard_firebox/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.4 name: watchguard_firebox title: WatchGuard Firebox -version: 0.1.0 +version: 0.1.1 description: Collect logs from WatchGuard Firebox with Elastic Agent. type: integration categories: From 2bb8b169f7e1f789d3fd97729fce4c0679f35447 Mon Sep 17 00:00:00 2001 From: peterydzynski <25185548+peterydzynski@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:03:15 -0400 Subject: [PATCH 02/13] [zeek] Add source/destination port 0 check to community id processor (#10205) - Added a check on the community_id processor in the Zeek connection pipeline to ensure source and destination port are not 0. --- packages/zeek/changelog.yml | 5 ++ .../_dev/test/pipeline/test-conn.log | 3 +- .../test/pipeline/test-conn.log-expected.json | 73 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 2 +- packages/zeek/manifest.yml | 2 +- 5 files changed, 82 insertions(+), 3 deletions(-) diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index 30a636f5946..edd6b1be1d8 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.24.2" + changes: + - description: Add source/destination port = 0 check to community_id processor. + type: bugfix + link: https://github.com/elastic/integrations/pull/10205 - version: "2.24.1" changes: - description: Add null checks to date processors in ntp pipeline. diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log index 73bf06d9817..9fccf6e0034 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log @@ -15,4 +15,5 @@ {"ts":1617062400.703851,"uid":"ChUxTmYLG37oO5qUb","id.orig_h":"10.156.0.2","id.orig_p":44942,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"ts":1617062400.704467,"uid":"CpeAOT3B11CTXJgzw2","id.orig_h":"10.156.0.2","id.orig_p":44946,"id.resp_h":"169.254.169.254","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} {"preview":false,"offset":0,"result":{"_bkt":"main~0~0758E7C3-1D0C-4B2B-8CF0-682BFEA86CDC","_cd":"0:12","_indextime":"1608752616","_raw":"{\"ts\":1547188417.857497,\"uid\":\"CAcJw21BbVedgFnYH5\",\"id.orig_h\":\"89.160.20.156\",\"id.orig_p\":38334,\"id.resp_h\":\"89.160.20.156\",\"id.resp_p\":53,\"proto\":\"udp\",\"service\":\"dns\",\"duration\":0.076967,\"orig_bytes\":75,\"resp_bytes\":178,\"conn_state\":\"SF\",\"local_orig\":false,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"Dd\",\"orig_pkts\":1,\"orig_ip_bytes\":103,\"resp_pkts\":1,\"resp_ip_bytes\":206,\"tunnel_parents\":[]}","_serial":"0","_si":["b590508aafed","main"],"_sourcetype":"access_log-too_small","_time":"2020-12-23 19:43:35.000 UTC","host":"Lees-MBP.localdomain","index":"main","linecount":"1","my_max":"1608759317","source":"/usr/local/var/log/httpd/access_log","sourcetype":"access_log-too_small","splunk_server":"b590508aafed"}} -{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.16.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} \ No newline at end of file +{"ts":"2021-06-09T20:55:13.160328Z","uid":"C2KP1V3alRLoxl4JB9","id.orig_h":"10.0.2.15","id.orig_p":46408,"id.resp_h":"172.16.9.68","id.resp_p":80,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"C","orig_pkts":0,"orig_ip_bytes":0,"resp_pkts":0,"resp_ip_bytes":0} +{"ts":1718280877.013007,"uid":"CL0jWq3WeMU4py67t7","id.orig_h":"10.2.4.200","id.orig_p":0,"id.resp_h":"175.16.199.74","id.resp_p":0,"proto":"tcp","conn_state":"OTH","local_orig":true,"local_resp":false,"missed_bytes":0,"history":"R","orig_pkts":1,"orig_ip_bytes":40,"resp_pkts":0,"resp_ip_bytes":0} \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json index 4b37e85ce9c..1f9748bd91e 100644 --- a/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json +++ b/packages/zeek/data_stream/connection/_dev/test/pipeline/test-conn.log-expected.json @@ -1327,6 +1327,79 @@ }, "session_id": "C2KP1V3alRLoxl4JB9" } + }, + { + "@timestamp": "2024-06-13T12:14:37.013Z", + "destination": { + "address": "175.16.199.74", + "bytes": 0, + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.74", + "packets": 0, + "port": 0 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "id": "CL0jWq3WeMU4py67t7", + "kind": "event", + "original": "{\"ts\":1718280877.013007,\"uid\":\"CL0jWq3WeMU4py67t7\",\"id.orig_h\":\"10.2.4.200\",\"id.orig_p\":0,\"id.resp_h\":\"175.16.199.74\",\"id.resp_p\":0,\"proto\":\"tcp\",\"conn_state\":\"OTH\",\"local_orig\":true,\"local_resp\":false,\"missed_bytes\":0,\"history\":\"R\",\"orig_pkts\":1,\"orig_ip_bytes\":40,\"resp_pkts\":0,\"resp_ip_bytes\":0}", + "type": [ + "connection", + "info" + ] + }, + "network": { + "bytes": 40, + "direction": "outbound", + "packets": 1, + "transport": "tcp" + }, + "related": { + "ip": [ + "10.2.4.200", + "175.16.199.74" + ] + }, + "source": { + "address": "10.2.4.200", + "bytes": 40, + "ip": "10.2.4.200", + "packets": 1, + "port": 0 + }, + "tags": [ + "preserve_original_event", + "local_orig", + "local_resp" + ], + "zeek": { + "connection": { + "history": "R", + "local_orig": true, + "local_resp": false, + "missed_bytes": 0, + "state": "OTH", + "state_message": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)." + }, + "session_id": "CL0jWq3WeMU4py67t7" + } } ] } \ No newline at end of file diff --git a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml index d71a6f883c6..afd14b56cb8 100644 --- a/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zeek/data_stream/connection/elasticsearch/ingest_pipeline/default.yml @@ -127,7 +127,7 @@ processors: copy_from: destination.address if: ctx.destination?.address != null - community_id: - if: 'ctx.network?.transport != "icmp"' + if: 'ctx.network?.transport != "icmp" && ctx.source?.port != 0 && ctx.destination?.port != 0' - community_id: icmp_type: zeek.connection.icmp.type icmp_code: zeek.connection.icmp.code diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index 024f5cbf872..5399028a440 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek -version: "2.24.1" +version: "2.24.2" description: Collect logs from Zeek with Elastic Agent. type: integration icons: From acaff2ca9081f27d705d73d90b8adfd4945ee916 Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Wed, 7 Aug 2024 09:20:26 -0500 Subject: [PATCH 03/13] [fortinet_fortiproxy] Remap devname to observer.name and process url field (#10679) - Remap the devname vendor field to observer.name - Remap the url vendor field to url.original and run through uri_parts processor --- packages/fortinet_fortiproxy/changelog.yml | 5 ++ .../pipeline/test-example.log-expected.json | 84 ++++++++++++------- .../elasticsearch/ingest_pipeline/default.yml | 8 +- .../data_stream/log/fields/ecs.yml | 24 ++++++ packages/fortinet_fortiproxy/docs/README.md | 14 ++++ packages/fortinet_fortiproxy/manifest.yml | 2 +- 6 files changed, 107 insertions(+), 30 deletions(-) diff --git a/packages/fortinet_fortiproxy/changelog.yml b/packages/fortinet_fortiproxy/changelog.yml index 92d41de3769..5cd61e64a2f 100644 --- a/packages/fortinet_fortiproxy/changelog.yml +++ b/packages/fortinet_fortiproxy/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.3.0" + changes: + - description: Remap devname to observer.name and process url field. + type: enhancement + link: https://github.com/elastic/integrations/pull/10679 - version: "0.2.0" changes: - description: Add tags.yml file so that integration's dashboards and saved searches are tagged with "Security Solution" and displayed in the Security Solution UI. Expanded categories. diff --git a/packages/fortinet_fortiproxy/data_stream/log/_dev/test/pipeline/test-example.log-expected.json b/packages/fortinet_fortiproxy/data_stream/log/_dev/test/pipeline/test-example.log-expected.json index 3866c7f0a34..55f60d58047 100644 --- a/packages/fortinet_fortiproxy/data_stream/log/_dev/test/pipeline/test-example.log-expected.json +++ b/packages/fortinet_fortiproxy/data_stream/log/_dev/test/pipeline/test-example.log-expected.json @@ -361,12 +361,12 @@ "name": "external" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "internal" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -493,12 +493,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -616,12 +616,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -737,12 +737,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -776,6 +776,12 @@ "packets": 0, "port": 40946 }, + "url": { + "domain": "google.com", + "original": "https://google.com/", + "path": "/", + "scheme": "https" + }, "user_agent": { "device": { "name": "Other" @@ -869,12 +875,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -908,6 +914,12 @@ "packets": 0, "port": 57748 }, + "url": { + "domain": "steampowered.com", + "original": "https://steampowered.com/", + "path": "/", + "scheme": "https" + }, "user_agent": { "device": { "name": "Other" @@ -997,12 +1009,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1036,6 +1048,12 @@ "packets": 0, "port": 36834 }, + "url": { + "domain": "github.com", + "original": "https://github.com/", + "path": "/", + "scheme": "https" + }, "user_agent": { "device": { "name": "Other" @@ -1137,7 +1155,7 @@ "bytes": 290 }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1177,6 +1195,8 @@ }, "url": { "domain": "google.com", + "original": "https://google.com/", + "path": "/", "scheme": "https" }, "user_agent": { @@ -1270,7 +1290,7 @@ "bytes": 82 }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1301,6 +1321,8 @@ }, "url": { "domain": "google.com", + "original": "https://google.com/", + "path": "/", "scheme": "https" }, "user_agent": { @@ -1394,7 +1416,7 @@ "bytes": 743 }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1425,6 +1447,8 @@ }, "url": { "domain": "google.com", + "original": "https://google.com/", + "path": "/", "scheme": "https" }, "user_agent": { @@ -1518,7 +1542,7 @@ "bytes": 80 }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1549,6 +1573,8 @@ }, "url": { "domain": "adobe.com", + "original": "https://adobe.com/", + "path": "/", "scheme": "https" }, "user_agent": { @@ -1642,7 +1668,7 @@ "bytes": 88 }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1673,6 +1699,8 @@ }, "url": { "domain": "www.adobe.com", + "original": "https://www.adobe.com/", + "path": "/", "scheme": "https" }, "user_agent": { @@ -1736,7 +1764,7 @@ }, "message": "Performance statistics: average CPU: 0, memory: 29, concurrent sessions: 119, setup-rate: 0", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1783,7 +1811,7 @@ }, "message": "failed to send urlfilter packet", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1829,7 +1857,7 @@ }, "message": "interface port1 gets a DHCP lease, ip:10.0.128.2, mask:255.255.255.255, gateway:10.0.128.1, lease expires:Tue May 7 10:11:16 2024", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1900,7 +1928,7 @@ }, "message": "Administrator Admin login failed from https(175.16.199.42) because of invalid user name", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -1965,7 +1993,7 @@ }, "message": "Fortiproxyupdate now fsci=yes from 175.16.199:443", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2019,7 +2047,7 @@ }, "message": "Edit firewall.policy 1", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2072,7 +2100,7 @@ }, "message": "Delete firewall.policy 3", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2126,7 +2154,7 @@ }, "message": "Add firewall.policy 2", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2179,7 +2207,7 @@ }, "message": "Move firewall.policy 2 to 1", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2233,7 +2261,7 @@ } }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2287,7 +2315,7 @@ } }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2341,7 +2369,7 @@ } }, "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2388,7 +2416,7 @@ }, "message": "Attempt to add tag FCTEMS_ALL_FORTICLOUD_SERVERS failed. Code (-2147483646)", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2459,12 +2487,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2569,12 +2597,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2658,7 +2686,7 @@ }, "message": "Performance statistics: average CPU: 0, memory: 29, concurrent sessions: 38, setup-rate: 1", "observer": { - "hostname": "TEST-PXY01", + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", @@ -2749,12 +2777,12 @@ "name": "port1" } }, - "hostname": "TEST-PXY01", "ingress": { "interface": { "name": "port2" } }, + "name": "TEST-PXY01", "product": "FortiProxy", "serial_number": "FPXTESTPXY01", "type": "proxy", diff --git a/packages/fortinet_fortiproxy/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/fortinet_fortiproxy/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 1177d6894d1..071c27e652a 100644 --- a/packages/fortinet_fortiproxy/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/fortinet_fortiproxy/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -351,7 +351,7 @@ processors: - rename: tag: rename_devname field: _fields_.devname - target_field: observer.hostname + target_field: observer.name ignore_missing: true - rename: tag: rename_direction @@ -563,6 +563,12 @@ processors: field: client.ip if: ctx._fields_.clientip != null + - uri_parts: + tag: process_url + field: _fields_.url + keep_original: true + ignore_missing: true + # ------------------------------------------------------------------------------ # Cleanup. diff --git a/packages/fortinet_fortiproxy/data_stream/log/fields/ecs.yml b/packages/fortinet_fortiproxy/data_stream/log/fields/ecs.yml index 3bdec85a6d6..c3a99e9bf5d 100644 --- a/packages/fortinet_fortiproxy/data_stream/log/fields/ecs.yml +++ b/packages/fortinet_fortiproxy/data_stream/log/fields/ecs.yml @@ -208,6 +208,30 @@ name: url.domain - external: ecs name: url.scheme +- external: ecs + name: url.extension +- external: ecs + name: url.original +- external: ecs + name: url.path +- external: ecs + name: url.fragment +- external: ecs + name: url.port +- external: ecs + name: url.query +- external: ecs + name: url.username +- external: ecs + name: url.password +- external: ecs + name: url.subdomain +- external: ecs + name: url.top_level_domain +- external: ecs + name: url.full +- external: ecs + name: url.registered_domain - external: ecs name: user_agent.device.name - external: ecs diff --git a/packages/fortinet_fortiproxy/docs/README.md b/packages/fortinet_fortiproxy/docs/README.md index 5b1a74cf030..1b60f01d667 100644 --- a/packages/fortinet_fortiproxy/docs/README.md +++ b/packages/fortinet_fortiproxy/docs/README.md @@ -888,7 +888,21 @@ An example event for `log` looks as following: | tags | List of keywords used to tag each event. | keyword | | threat.feed.name | The name of the threat feed in UI friendly format. | keyword | | url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | +| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | +| url.full.text | Multi-field of `url.full`. | match_only_text | +| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| url.original.text | Multi-field of `url.original`. | match_only_text | +| url.password | Password of the request. | keyword | +| url.path | Path of the request, such as "/search". | wildcard | +| url.port | Port of the request, such as 443. | long | +| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | | url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | +| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | +| url.username | Username of the request. | keyword | | user_agent.device.name | Name of the device. | keyword | | user_agent.name | Name of the user agent. | keyword | | user_agent.original | Unparsed user_agent string. | keyword | diff --git a/packages/fortinet_fortiproxy/manifest.yml b/packages/fortinet_fortiproxy/manifest.yml index f75d6789bd9..0e307236d54 100644 --- a/packages/fortinet_fortiproxy/manifest.yml +++ b/packages/fortinet_fortiproxy/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.3 name: fortinet_fortiproxy title: "Fortinet FortiProxy" -version: 0.2.0 +version: 0.3.0 description: "Collect logs from Fortinet FortiProxy with Elastic Agent." type: integration categories: From bc599975a932b5acb398cc6d5e2e306bf1801817 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:47:35 +0200 Subject: [PATCH 04/13] Bump golang.org/x/tools from 0.23.0 to 0.24.0 (#10723) Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.23.0 to 0.24.0. - [Release notes](https://github.com/golang/tools/releases) - [Commits](https://github.com/golang/tools/compare/v0.23.0...v0.24.0) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 014895beeb4..179308577b6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/magefile/mage v1.15.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.9.0 - golang.org/x/tools v0.23.0 + golang.org/x/tools v0.24.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -174,14 +174,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/mod v0.19.0 // indirect - golang.org/x/net v0.27.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.171.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 08f661123c4..157f212cccb 100644 --- a/go.sum +++ b/go.sum @@ -533,8 +533,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -545,8 +545,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -563,8 +563,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= @@ -576,8 +576,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -603,16 +603,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -621,8 +621,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -637,8 +637,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 7025e89aea70ea32c86dd730c0076322ac0c0769 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Wed, 7 Aug 2024 18:23:22 +0200 Subject: [PATCH 05/13] [Docs] Add ecs@mappings migration guide for integration developers (#10079) Add history, usage, and recommendations for using the ecs@mappings component template in integrations. --------- Co-authored-by: Jaime Soriano Pastor --- docs/ecs@mappings_migration_guide.md | 318 +++++++++++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 docs/ecs@mappings_migration_guide.md diff --git a/docs/ecs@mappings_migration_guide.md b/docs/ecs@mappings_migration_guide.md new file mode 100644 index 00000000000..2f536b1d52d --- /dev/null +++ b/docs/ecs@mappings_migration_guide.md @@ -0,0 +1,318 @@ +# ecs@mappings migration guide for integration developers + +## History + +In the initial stages, our approach involved individually specifying [ECS](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) fields within each package. + +```yaml +- name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. +``` + +As we progressed, the need for more efficient methodologies became apparent, prompting us to explore alternative strategies. + +**How are integrations handling ECS mappings today?** + +Today, integrations employ one of two strategies to manage ECS mappings: + +- Referencing ECS mappings (the predominant method) +- Importing ECS mappings (a smaller subset, approximately 40 integrations, opt to import ECS mappings directly) + +### Referencing ECS fields + +Define external dependency: + +```yaml +# packages/azure/_dev/build/build.yml + +dependencies: + ecs: + reference: git@v8.11.0 +``` + +Developers can reference the external definition: + +```yaml +# packages/azure/data_stream/activitylogs/fields/agent.yml + +- name: cloud.provider + external: ecs +``` + +#### Consequences + +Even if each field references the external definition, integration developers must list all fields in various `.yml` files. + +### Importing ECS mappings + +With elastic-package [v0.71.0](https://github.com/elastic/elastic-package/releases/tag/v0.71.0) and package-spec 2.3.0, we [added](https://github.com/elastic/elastic-package/pull/1073) the option of importing the ECS mappings during the package build to avoid explicitly listing all the fields. + +When we set `import_mappings: true` in the `_dev/build/build.yml` file, elastic-package fetches the static [ecs_mappings.yml](https://github.com/elastic/elastic-package/blob/a44250eda089f89cc820c0ba5492bef71857aeb1/internal/builder/_static/ecs_mappings.yaml) file and embeds its content in the `logs-azure.eventhub@package` component template. + +```yaml +# packages/azure_blob_storage/_dev/build/build.yml + +dependencies: + ecs: + reference: "git@v8.11.0" + import_mappings: true +``` + +With `import_mappings: true`, the package doesn’t need to define ECS fields. + +```yaml +# packages/azure_blob_storage/data_stream/generic/fields/iamnotneeded.yml + +¯\_(ツ)_/¯ +``` + +See [Custom Azure Blob Storage Input](https://github.com/elastic/integrations/tree/main/packages/azure_blob_storage) as an example of integrations importing ECS mappings. + +#### Consequences + +- There is no need to define ECS fields \o/ +- ECS field definitions come from one place, the static [ecs_mappings.yml](https://github.com/elastic/elastic-package/blob/a44250eda089f89cc820c0ba5492bef71857aeb1/internal/builder/_static/ecs_mappings.yaml) file in elastic-package sources. +- However, setting up elastic-package has a maintenance cost of keeping [ecs_mappings.yml](https://github.com/elastic/elastic-package/blob/a44250eda089f89cc820c0ba5492bef71857aeb1/internal/builder/_static/ecs_mappings.yaml) up-to-date with changes in ECS. + +## Why change? + +A new opportunity to improve our handling of ECS mappings appeared when Elasticsearch v8.9.0 [introduced](https://github.com/elastic/elasticsearch/issues/95538) the new [ecs@mappings](https://github.com/elastic/elasticsearch/blob/b4938e16457dc69d392235eaf404a6dad9ddb717/x-pack/plugin/core/template-resources/src/main/resources/ecs%40mappings.json) component template to `logs-*-*` index template. + +With the [ecs@mappings](https://github.com/elastic/elasticsearch/blob/b4938e16457dc69d392235eaf404a6dad9ddb717/x-pack/plugin/core/template-resources/src/main/resources/ecs%40mappings.json) component template, we have an official and maintained definition of ECS mappings template. + +However, Fleet v8.9.0 did not include the [ecs@mappings](https://github.com/elastic/elasticsearch/blob/b4938e16457dc69d392235eaf404a6dad9ddb717/x-pack/plugin/core/template-resources/src/main/resources/ecs%40mappings.json) component template in index templates for integrations. + +From stack v8.13.0, Fleet will [include](https://github.com/elastic/kibana/issues/174905) ecs@mappings component templates in all integrations, making it easier for integration users and developers to access logs and metrics data streams. + +#### Consequences + +- ecs@mappings from Elasticsearch are the single source of truth for ECS mappings. +- ECS mappings are available and out-of-the-box; there is no need to import or reference external mapping. + +## How to start using ecs@mappings + +### Requirements + +Before starting to leverage only the ecs@mappings component template for ECS mappings in your integration package, you need to meet the following requirements: + +- The minimum stack version must be 8.13.0. +- The minimum elastic-package version must be 0.99.0. + +#### Why elastic-package version 0.99.0+? + +When your integration package only supports stack versions 8.13.0+, it validates the field definitions using the fields schema from the ECS repo on sample_event.json and test documents at + +```text +packages/azure/data_stream/activitylogs/_dev/test/pipeline/ +``` + +For example, `elastic-package` fetches the field definitions for ECS 8.11.0 at: + +https://raw.githubusercontent.com/elastic/ecs/v8.11.0/generated/ecs/ecs_nested.yml + + +### Migration Paths + +Here is a list of known migration paths from referencing external fields and importing the legacy ECS mappings. + +#### From Referencing ECS fields + +You can start by removing references to external definitions and running tests. You should consider a few aspects while migrating from referencing ECS to the ecs@mappings component template. + +##### Check your pipeline test coverage + +Good coverage in _dev/test/pipeline/ tests is essential for catching problems. Consider adding more sample documents to increase the chances of catching problems. + +##### Existing tests may start to fail + +For Integrations that target stack 8.13+, elastic-package 0.99 also brings an additional schema validation that can uncover inconsistencies. + +For example, by enabling ecs@mappings in Azure Logs, we learned that the current "event.outcome" field value, “succeeded,” is not one of the expected values (it must be between “success”, “failure”, and “unknown”). + +##### Take underlying assumptions into account + +The ecs@mappings expect that logs and metrics shippers (and the related pipelines, if any) emit field values using the correct field type. + +For example, if you send a document with a boolean field: + +```json +{ + "coldstart": true +} +``` + +Both legacy and modern ECS mappings will map the field as a boolean field type. + +However, if your logs source emits something like this document: + +```json +{ + "coldstart": "true" +} +``` + +The modern ecs@mappings will not coerce the value and map this field as a keyword. + +We can consider this an edge case. However, it can happen, even if it looks weird. Personally, I had spotted cases like this in one of the major CSPs. I suggest dealing with edge cases from your logs or metrics source using the @custom pipeline or mappings. + +Each approach to ECS mappings has its own tradeoffs. If you want to learn more about the one we picked and what other options we considered, you can read https://github.com/elastic/elasticsearch/issues/85146#issuecomment-2031285084 + +#### From Importing ECS fields + +Integration packages importing legacy ECS mappings do not have field definitions. The transition should be more accessible. + +When the min stack version is ^8.13.0, you can stop importing the legacy mappings: + +```yaml +# packages/azure_blob_storage/_dev/build/build.yml + +dependencies: + ecs: + reference: "git@v8.11.0" + import_mappings: true # remove this line, default is false. +``` + +Good `sample_event.json` and test documents are essential. + +### Existing approaches to define mappings will continue to work + +In package-spec 3.1.3, we deprecated the use of import_mappings: true. Importing is no longer the recommended way to deal with ECS mappings. + +Since the package owners may want to keep the minimum stack version < 8.13, all existing approaches to define mappings will continue to work. + +We recommend migrating to ecs@mappings to reap the benefits of centralized and up-to-date ECS field definitions. + +#### package-spec recommendations + +Consider upgrading to the recent package-spec according to your minimum stack requirements. The benefits (especially additional checks that elastic-package delivers) outweigh the costs. + +### Override, if required + +The ecs@mappings can deal with ECS mappings in all standard cases. + +However, integration developers can continue using the field definition of specific fields to override the definition in Elasticsearch if needed. + +## Q&A + +Here are a few topic and questions people asked when we started rolling out the `ecs@mappings` component template in integrations. + +### TSDB fields in metrics data streams + +ECS field definitions in Elasticsearch do not include TSDB settings like dimensions. Developers can add a field definition with the additional dimension setting when needed. + +However, this will no longer be needed when integrations are OpenTelemetry-based. That’s because all attributes and resource attributes will be dimensions by default. + +### How can I learn which ECS version a given stack version supports? + +#### Question + +For example, if I am running 8.13.0, which ECS version does the 8.13.0 `ecs@mappings` component template support? + +#### Answer + +The `ecs@mappings` component template in each stack version supports the ECS version available at the time of the stack release. + +An automated test verifies daily that `ecs@mappings` don’t miss any ECS field. + +As Eyal explained: + +> “It fetches the current state of all fields from the ECS repo, creates test documents that contain an example for each field, and verifies that an index that relies on the dynamic templates will contain all the right mappings when indexing the test documents.” + +### Are new versions of ecs@mappings retro-compatible? + +#### Question + +For example, suppose I am on 8.13.0, and the integration validates ECS fields using the latest ECS v8.11.0. + +What are the chances that future stack versions (8.14, 8.15, etc) may ship with an ecs@mappings component template that changes the integration's behavior? What can I do to prevent this from happening or detect it in advance? + +#### Answer + +Since we based the `ecs@mappings` component template on pattern matching, we expect little to no changes over time. + +New fields in ECS should receive a mapping, and automated tests are in place to ensure that the `ecs@mappings` component template adequately supports all ECS fields. + +Integration developers should target new versions of ECS in the “dependencies.ecs.reference” in their integration to let elastic-package check for compliance. + +The transition to Semantic Conventions (OTel) is more likely to introduce breaking changes than ECS updates. + +## Scenarios + +Here are scenario that may be affected by the introduction of `ecs@mappings` in integrations. + +### A user clones an integration index template to customize the ILM policy + +#### Description + +Suppose a user installs the 1Password integration on stack 8.12. + +Fleet creates the `logs-1password.audit_events` index template, with the `logs-1password.audit_events-*` index pattern, and the following component templates: + +```text +logs@settings +logs-1password.audit_events@package +logs-1password.audit_events@custom +.fleet_globals-1 +.fleet_agent_id_verification-1 +``` + +The user has three environments (dev, test, prod) and wants to use a distinct ILM policy in each environment. They decide to use a different namespace for each environment (a common practice in enterprise environments). + +The user finds https://www.elastic.co/guide/en/fleet/current/data-streams-ilm-tutorial.html#data-streams-ilm-one, and at step 3 they read the following steps: + +1. Navigate to **Stack Management > Index Management > Index Templates**. +2. Find the index template you want to clone. The index template will have the and in its name, but not the . In this case, it’s metrics-system.network. +3. Select **Actions > Clone**. +4. Set the name of the new index template to `metrics-system.network-production`. + +They clone the original index template three times and set up individual ILM policies. + +They end up with four index templates: + +- logs-1password.audit_events (original) +- logs-1password.audit_events-dev (includes logs-1password.audit_events-dev@custom) +- logs-1password.audit_events-test (includes logs-1password.audit_events-uat@custom) +- logs-1password.audit_events-production (includes logs-1password.audit_events-production@custom) + +Then, the user upgrades the stack from 8.12 to 8.13. + +After the upgrade, here’s each index template's list of component templates. + +The original index template gets the ecs@mappings component. + +```text +# logs-1password.audit_events (original) + +logs@settings +logs-1password.audit_events@package +logs-1password.audit_events@custom +ecs@mappings +.fleet_globals-1 +.fleet_agent_id_verification-1 +``` + +The cloned index template is unchanged: + +```text +# logs-1password.audit_events-dev + +logs@settings +logs-1password.audit_events@package +logs-1password.audit_events@custom +.fleet_globals-1 +.fleet_agent_id_verification-1 +``` + +#### Description + +We couldn’t identify an actual solution to address this scenario. + +To mitigate potential issues from this scenario, we are currently extending the information available to end users: + +- The “Notable changes” section in the Fleet 8.13 release notes. +- Created the KB article [Potential ecs@mappings issue for index template clones on 8.13+](https://support.elastic.dev/knowledge/view/df0eaa25) +- Updated the [Tutorial: Customize data retention policies document](https://www.elastic.co/guide/en/fleet/current/data-streams-ilm-tutorial.html) with a note and instructions to update index templates cloned before Elasticsearch 8.13 + From df846beb3b79e0c6a9e4ebe30324ccf493f42384 Mon Sep 17 00:00:00 2001 From: Florian Lehner Date: Wed, 7 Aug 2024 18:36:36 +0200 Subject: [PATCH 06/13] Profiling: Update requirements (#10724) - Minimum supported kernel version is 4.19 - With 8.10 Profiling did become GA - versions prior to 8.10 are not fully supported. --- packages/universal_profiling_agent/changelog.yml | 5 +++++ packages/universal_profiling_agent/docs/README.md | 4 ++-- packages/universal_profiling_agent/manifest.yml | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/universal_profiling_agent/changelog.yml b/packages/universal_profiling_agent/changelog.yml index d397b35de69..d889653927c 100644 --- a/packages/universal_profiling_agent/changelog.yml +++ b/packages/universal_profiling_agent/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: 8.14.0 + changes: + - description: Update requirement text + type: enhancement + link: https://github.com/elastic/integrations/pull/10724 - version: 8.13.2 changes: - description: Add disable_tls option diff --git a/packages/universal_profiling_agent/docs/README.md b/packages/universal_profiling_agent/docs/README.md index d73a95a7e9f..000f4ef5eeb 100644 --- a/packages/universal_profiling_agent/docs/README.md +++ b/packages/universal_profiling_agent/docs/README.md @@ -5,8 +5,8 @@ Get a comprehensive understanding of what lines of code are consuming compute resources throughout your entire fleet by visualizing your data in Kibana using the flamegraph, stacktraces, and top functions views. ## Requirements -* The workloads to be profiled must be running on Linux machines; with kernel >=4.15 -* Elastic Cloud, version 8.7 or higher +* The workloads to be profiled must be running on Linux machines; The minimum supported kernel version is either 4.19 for x86_64 or 5.5 for ARM64 machines. +* Elastic Cloud, version 8.10 or higher ## Key Features diff --git a/packages/universal_profiling_agent/manifest.yml b/packages/universal_profiling_agent/manifest.yml index 9f8d53f1db3..d51e0fcdb6f 100644 --- a/packages/universal_profiling_agent/manifest.yml +++ b/packages/universal_profiling_agent/manifest.yml @@ -1,6 +1,6 @@ name: profiler_agent title: Universal Profiling Agent -version: 8.13.2 +version: 8.14.0 categories: ["elastic_stack", "monitoring"] description: Fleet-wide, whole-system, continuous profiling with zero instrumentation. conditions: From 80576ff924ad84e24dc6e9561d19673fca74387c Mon Sep 17 00:00:00 2001 From: peterydzynski <25185548+peterydzynski@users.noreply.github.com> Date: Wed, 7 Aug 2024 20:22:53 -0400 Subject: [PATCH 07/13] m365_defender: set network.transport to ssl for ssl type actions (#10730) Co-authored-by: Peter Rydzynski --- packages/m365_defender/changelog.yml | 5 +++++ .../event/_dev/test/pipeline/test-device.log-expected.json | 2 +- .../event/elasticsearch/ingest_pipeline/pipeline_device.yml | 2 +- packages/m365_defender/manifest.yml | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/m365_defender/changelog.yml b/packages/m365_defender/changelog.yml index addcb64ae84..108204c42ae 100644 --- a/packages/m365_defender/changelog.yml +++ b/packages/m365_defender/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.14.3" + changes: + - description: Fix sslconnectioninspected event `network.protocol` getting set to `dns`. + type: bugfix + link: https://github.com/elastic/integrations/pull/10730 - version: "2.14.2" changes: - description: Fix `network.transport` and `network.protocol` processing. diff --git a/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json b/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json index 3732a6ba2f9..4a1b0016714 100644 --- a/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json +++ b/packages/m365_defender/data_stream/event/_dev/test/pipeline/test-device.log-expected.json @@ -3128,7 +3128,7 @@ }, "network": { "direction": "outbound", - "protocol": "dns", + "protocol": "ssl", "transport": "tcp" }, "process": { diff --git a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml index 2859db7a3cf..fe28885875e 100644 --- a/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml +++ b/packages/m365_defender/data_stream/event/elasticsearch/ingest_pipeline/pipeline_device.yml @@ -2389,7 +2389,7 @@ processors: override: true - set: field: network.protocol - value: dns + value: ssl tag: set_network_protocol_ssl if: ctx.m365_defender?.event?.action?.type != null && ctx.m365_defender.event.action.type.toLowerCase().contains('ssl') override: true diff --git a/packages/m365_defender/manifest.yml b/packages/m365_defender/manifest.yml index 6105247491a..d5336220e2f 100644 --- a/packages/m365_defender/manifest.yml +++ b/packages/m365_defender/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: m365_defender title: Microsoft M365 Defender -version: "2.14.2" +version: "2.14.3" description: Collect logs from Microsoft M365 Defender with Elastic Agent. categories: - "security" From 16b2eeabf9bc6c0b2054c27029c0ec203e8e850a Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Reddy Burri Date: Thu, 8 Aug 2024 10:37:50 +0530 Subject: [PATCH 08/13] sophos_central: Update docs for token_url configuration (#10720) Update docs for `token_url` configuration. [The input](https://github.com/elastic/integrations/blob/main/packages/sophos_central/data_stream/alert/agent/stream/httpjson.yml.hbs#L9) already appends the token url path `/api/v2/oauth2/token` to the `auth.oauth2.token_url`, needing the user to configure only the base url for token_url parameter. For example: `https://id.sophos.com`. The current README doc doesn't indicate that the url path needs to be removed when configuring token_url. This can lead to duplicate url path in the `auth.oauth2.token_url`, for example: `https://id.sophos.com/api/v2/oauth2/token/api/v2/oauth2/token`. The PR addresses this by updating the README doc. --- packages/sophos_central/_dev/build/docs/README.md | 2 +- packages/sophos_central/changelog.yml | 5 +++++ packages/sophos_central/docs/README.md | 2 +- packages/sophos_central/manifest.yml | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/sophos_central/_dev/build/docs/README.md b/packages/sophos_central/_dev/build/docs/README.md index b6da4f09fdd..3a0e2b323f4 100644 --- a/packages/sophos_central/_dev/build/docs/README.md +++ b/packages/sophos_central/_dev/build/docs/README.md @@ -33,7 +33,7 @@ The Elastic Integration for Sophos Central requires the following Authentication - Grant Type - Scope - Tenant ID - - Token URL + - Token URL (without the URL path) **NOTE**: Sophos central supports logs only upto last 24 hrs. diff --git a/packages/sophos_central/changelog.yml b/packages/sophos_central/changelog.yml index be4a3c60012..cc4109e2e76 100644 --- a/packages/sophos_central/changelog.yml +++ b/packages/sophos_central/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.0" + changes: + - description: Update docs for token_url configuration. + type: enhancement + link: https://github.com/elastic/integrations/pull/10720 - version: "1.15.0" changes: - description: Removed import_mappings. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. diff --git a/packages/sophos_central/docs/README.md b/packages/sophos_central/docs/README.md index 6e5d2c47554..43700ab7056 100644 --- a/packages/sophos_central/docs/README.md +++ b/packages/sophos_central/docs/README.md @@ -33,7 +33,7 @@ The Elastic Integration for Sophos Central requires the following Authentication - Grant Type - Scope - Tenant ID - - Token URL + - Token URL (without the URL path) **NOTE**: Sophos central supports logs only upto last 24 hrs. diff --git a/packages/sophos_central/manifest.yml b/packages/sophos_central/manifest.yml index aef6d3a8b9e..a20a1f48483 100644 --- a/packages/sophos_central/manifest.yml +++ b/packages/sophos_central/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: sophos_central title: Sophos Central -version: "1.15.0" +version: "1.16.0" description: This Elastic integration collects logs from Sophos Central with Elastic Agent. type: integration categories: @@ -73,7 +73,7 @@ policy_templates: - name: token_url type: text title: Token URL - description: "Token_url must be the same as used while generating tenant_id, follow this link(https://developer.sophos.com/getting-started-tenant) for configuration." + description: Token_url must be the same as used while generating tenant_id, follow this [link](https://developer.sophos.com/getting-started-tenant) for configuration. This URL should be without the url path, for example - `https://id.sophos.com` i.e., without the path `/api/v2/oauth2/token`. multi: false required: true show_user: false From a436aa74b23564c4710620cc08ad21f80b31821d Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 8 Aug 2024 10:39:40 +0200 Subject: [PATCH 09/13] Populate `event.outcome` based on `sso_token_success`, when present (#10697) The `sso_token_success` field isn't in the API documentation, but we do have a field definition for it and we see it returned with `"event_type": "sso_auth"` (without the `success` field). --- packages/jumpcloud/changelog.yml | 5 +++++ .../events/elasticsearch/ingest_pipeline/default.yml | 8 ++++++++ packages/jumpcloud/manifest.yml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/jumpcloud/changelog.yml b/packages/jumpcloud/changelog.yml index 9e0f151bbb5..9194a366055 100644 --- a/packages/jumpcloud/changelog.yml +++ b/packages/jumpcloud/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.12.0" + changes: + - description: Populate 'event.outcome' based on 'sso_token_success', when present + type: enhancement + link: https://github.com/elastic/integrations/pull/10697 - version: "1.11.0" changes: - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. diff --git a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml index dbe587d9818..f1f5eebf86a 100644 --- a/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml +++ b/packages/jumpcloud/data_stream/events/elasticsearch/ingest_pipeline/default.yml @@ -161,6 +161,14 @@ processors: - set: field: event.outcome value: unknown + - set: + field: event.outcome + value: success + if: ctx.jumpcloud?.event?.sso_token_success == true + - set: + field: event.outcome + value: failure + if: ctx.jumpcloud?.event?.sso_token_success == false - set: field: event.outcome value: success diff --git a/packages/jumpcloud/manifest.yml b/packages/jumpcloud/manifest.yml index 81ffd8fd3c6..4ff6076c00e 100644 --- a/packages/jumpcloud/manifest.yml +++ b/packages/jumpcloud/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: jumpcloud title: "JumpCloud" -version: "1.11.0" +version: "1.12.0" description: "Collect logs from JumpCloud Directory as a Service" type: integration categories: From 1c2a9f2445bdc32e27c87039d990d15c7fa25a81 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Thu, 8 Aug 2024 18:51:59 +0930 Subject: [PATCH 10/13] f5_bigip: handle x_forwarded_for_header_value fields with multiple IP addresses (#10718) --- packages/f5_bigip/changelog.yml | 5 + .../test/pipeline/test-pipeline-bigip-asm.log | 1 + .../test-pipeline-bigip-asm.log-expected.json | 137 +++++++++++++++++- .../ingest_pipeline/pipeline_bigipasm.yml | 20 ++- .../data_stream/log/sample_event.json | 18 ++- packages/f5_bigip/docs/README.md | 18 ++- packages/f5_bigip/manifest.yml | 2 +- 7 files changed, 175 insertions(+), 26 deletions(-) diff --git a/packages/f5_bigip/changelog.yml b/packages/f5_bigip/changelog.yml index 0739ca0d947..9f1240dbbd3 100644 --- a/packages/f5_bigip/changelog.yml +++ b/packages/f5_bigip/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.19.0" + changes: + - description: Handle `x_forwarded_for_header_value` fields with multiple IP addresses. + type: enhancement + link: https://github.com/elastic/integrations/pull/10718 - version: "1.18.1" changes: - description: Update event.kind values based on severity. diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log index 52ff0f7582c..8c8ad45d3df 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log @@ -2,3 +2,4 @@ {"hostname":"hostname","management_ip_address":"10.0.1.4","management_ip_address_2":"","http_class_name":"/Common/app.app/app_policy","web_application_name":"/Common/app.app/app_policy","policy_name":"/Common/app.app/app_policy","policy_apply_date":"2018-11-19 22:17:57","violations":"Evasion technique detected","support_id":"1730614276869062795","request_status":"blocked","response_code":"0","ip_client":"192.168.0.1","route_domain":"0","method":"GET","protocol":"HTTP","query_string":"","x_forwarded_for_header_value":"192.168.0.1","sig_ids":"","sig_names":"","date_time":"2018-11-19 22:34:40","severity":"Critical","attack_type":"Detection Evasion,Path Traversal","geo_location":"US","ip_address_intelligence":"N/A","username":"N/A","session_id":"f609d8a924419638","src_port":"49804","dest_port":"80","dest_ip":"10.0.2.10","sub_violations":"Evasion technique detected:Directory traversals","virus_name":"N/A","violation_rating":"3","websocket_direction":"N/A","websocket_message_type":"N/A","device_id":"N/A","staged_sig_ids":"","staged_sig_names":"","threat_campaign_names":"","staged_threat_campaign_names":"","blocking_exception_reason":"N/A","captcha_result":"not_received","uri":"/directory/file","fragment":"","request":"GET /admin/..%2F..%2F..%2Fdirectory/file HTTP/1.0\\r\\nHost: host.westus.cloudapp.azure.com\\r\\nConnection: keep-alive\\r\\nCache-Control: max-age","tenant":"Common","application":"app.app","telemetryEventCategory":"ASM"} {"attack_type":"Test Attack","date_time":"2018-11-19 22:34:40","dest_ip":"10.160.77.77","dest_port":"80","geo_info":"info","headers":"Host: 231213","http_class":"/Common/Test","ip_addr_intelli":"host1","ip_client":"81.2.69.142","ip_route_domain":"example.com","is_trunct":"no","manage_ip_addr":"81.2.69.142","method":"POST","policy_apply_date":"2021-09-30 02:51:31","policy_name":"/Common/Test","protocol":"HTTP","query_string":"","req":"POST /login.php HTTP/1.1\\r\\nHost: 81.2.69.142","req_status":"passed","resp":"HTTP/1.1 302 Found\\r\\nDate: Tue, 05 Oct 2021 17:30:14 ","resp_code":"302","route_domain":"example.com","session_id":"ab32bda123","severity":"Informational","sig_ids":"1abcd23bdc","sig_names":"test","src_port":"49744","sub_violates":"Sub-violation","support_id":"5438760667957952540","unit_host":"hostname","uri":"/login.php","username":"Test User","violate_details":"This is a details.","violate_rate":"0","violations":"deny","virus_name":"abcd","x_fwd_hdr_val":"test","telemetryEventCategory":"ASM","hostname":"localhost.localdomain","tenant":"Common","microservice": "N/A","response": "Response logging disabled","sig_cves": "N/A","staged_sig_cves": "N/A","tap_event_id": "N/A","tap_vid": "N/A","vs_name": "/Common/Server1_DVWA"} {"compression_method":"test_method","client_type":"test_client","conviction_traps":"test","credential_stuffing_lookup_result":"pass","enforced_by":"test","enforcement_action":"test_action","epoch_time":"1665576701","ip_with_route_domain":"example.com","is_truncated":"","likely_false_positive_sig_ids":"12345678","login_result":"success","mobile_application_name":"test_application","mobile_application_version":"test1.1","operation_id":"12345","password_hash_prefix":"test","protocol_info":"test_info","sig_set_names":"test_sig_name","slot_number":"1234","staged_sig_set_names":"test_staged_sig_name","tap_requested_actions":"test_tap_action","tap_sent_token":"20334","tap_transaction_id":"12345","unit_hostname":"hostname","violation_details":"test_detail","telemetryEventCategory":"ASM"} +{"attack_type":"N/A","blocking_exception_reason":"N/A","captcha_result":"not_received","date_time":"2024-08-06T10:03:36.000Z","dest_ip":"10.30.4.56","dest_port":"443","device_id":"N/A","fragment":"","geo_location":"N/A","hostname":"f5qa","http_class_name":"/Common/asmpolicy_lampqa","ip_address_intelligence":"N/A","ip_client":"10.43.24.23","management_ip_address":"10.52.34.33","management_ip_address_2":"N/A","method":"HEAD","microservice":"N/A","originalRawData":"<134>Aug 6 12:03:36 f5qa ASM:unit_hostname=\"f5qa\",management_ip_address=\"10.52.34.33\",management_ip_address_2=\"N/A\",http_class_name=\"/Common/asmpolicy_lampqa\",web_application_name=\"/Common/asmpolicy_lampqa\",policy_name=\"/Common/asmpolicy_lampqa\",policy_apply_date=\"2024-07-02 16:43:55\",violations=\"N/A\",support_id=\"5410866668007843666\",request_status=\"passed\",response_code=\"200\",ip_client=\"10.43.24.23\",route_domain=\"0\",method=\"HEAD\",protocol=\"HTTPS\",query_string=\"\",x_forwarded_for_header_value=\"10.43.24.23, 10.43.24.23\",sig_ids=\"N/A\",sig_names=\"N/A\",date_time=\"2024-08-06 12:03:36\",severity=\"Informational\",attack_type=\"N/A\",geo_location=\"N/A\",ip_address_intelligence=\"N/A\",username=\"N/A\",session_id=\"7a60af492530220b\",src_port=\"50668\",dest_port=\"443\",dest_ip=\"10.30.4.56\",sub_violations=\"N/A\",virus_name=\"N/A\",violation_rating=\"0\",websocket_direction=\"N/A\",websocket_message_type=\"N/A\",device_id=\"N/A\",staged_sig_ids=\"\",staged_sig_names=\"\",threat_campaign_names=\"N/A\",staged_threat_campaign_names=\"N/A\",blocking_exception_reason=\"N/A\",captcha_result=\"not_received\",microservice=\"N/A\",tap_event_id=\"N/A\",tap_vid=\"N/A\",vs_name=\"/Common/vs_externalqa13_443\",sig_cves=\"N/A\",staged_sig_cves=\"N/A\",uri=\"/repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar\",fragment=\"\",request=\"HEAD /repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar HTTP/1.1\\r\\nCache-Control: no-cache, no-store\\r\\nPragma: no-cache\\r\\nHost: domain.gent\\r\\nConnection: Keep-Alive\\r\\nUser-Agent: Apache-Maven/3.9.7 (Java 17.0.12; Windows 11 10.0)\\r\\nAccept-Encoding: gzip,deflate\\r\\nAuthorization: Basic dc2VydmljZWZhY3Rvcnk6TE1YaHZwRUxhRjJodEFScWFQkkk=\\r\\nX-Forwarded-For: 10.43.24.23, 10.43.24.23\\r\\nX-Forwarded-Proto: https\\r\\n\\r\\n\",response=\"Response logging disabled\"","policy_apply_date":"2024-07-02 16:43:55","policy_name":"/Common/asmpolicy_lampqa","protocol":"HTTPS","query_string":"","request":"HEAD /repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar HTTP/1.1\\r\\nCache-Control: no-cache, no-store\\r\\nPragma: no-cache\\r\\nHost: domain.gent\\r\\nConnection: Keep-Alive\\r\\nUser-Agent: Apache-Maven/3.9.7 (Java 17.0.12; Windows 11 10.0)\\r\\nAccept-Encoding: gzip,deflate\\r\\nAuthorization: Basic dc2VydmljZWZhY3Rvcnk6TE1YaHZwRUxhRjJodEFScWFQkkk=\\r\\nX-Forwarded-For: 10.43.24.23, 10.43.24.24\\r\\nX-Forwarded-Proto: https\\r\\n\\r\\n","request_status":"passed","response":"Response logging disabled","response_code":"200","route_domain":"0","session_id":"7a60af492530220b","severity":"Informational","sig_cves":"N/A","sig_ids":"N/A","sig_names":"N/A","src_port":"50668","staged_sig_cves":"N/A","staged_sig_ids":"","staged_sig_names":"","staged_threat_campaign_names":"N/A","sub_violations":"N/A","support_id":"5410868666607846666","tap_event_id":"N/A","tap_vid":"N/A","telemetryEventCategory":"ASM","tenant":"Common","threat_campaign_names":"N/A","uri":"/repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar","username":"N/A","violation_rating":"0","violations":"N/A","virus_name":"N/A","vs_name":"/Common/vs_externalqa13_443","web_application_name":"/Common/asmpolicy_lampqa","websocket_direction":"N/A","websocket_message_type":"N/A","x_forwarded_for_header_value":"10.43.24.23, 10.43.24.24"} diff --git a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json index c83658b2a37..c477ce1c09d 100644 --- a/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json +++ b/packages/f5_bigip/data_stream/log/_dev/test/pipeline/test-pipeline-bigip-asm.log-expected.json @@ -126,7 +126,9 @@ "direction": "Test", "message_type": "test" }, - "x_forwarded_for_header_value": "81.2.69.144" + "x_forwarded_for_header_value": [ + "81.2.69.144" + ] } }, "host": { @@ -293,7 +295,9 @@ }, "violations": "Evasion technique detected", "web_application_name": "/Common/app.app/app_policy", - "x_forwarded_for_header_value": "192.168.0.1" + "x_forwarded_for_header_value": [ + "192.168.0.1" + ] } }, "host": { @@ -344,7 +348,6 @@ ], "url": { "domain": "host.westus.cloudapp.azure.com", - "extension": "/directory/file", "original": "http://host.westus.cloudapp.azure.com/admin/..%2F..%2F..%2Fdirectory/file", "path": "/admin/../../../directory/file", "scheme": "http" @@ -561,6 +564,134 @@ "preserve_original_event", "preserve_duplicate_custom_fields" ] + }, + { + "@timestamp": "2024-08-06T10:03:36.000Z", + "client": { + "ip": "10.43.24.23", + "port": 50668 + }, + "destination": { + "ip": "10.30.4.56", + "port": 443 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "kind": "event", + "original": "{\"attack_type\":\"N/A\",\"blocking_exception_reason\":\"N/A\",\"captcha_result\":\"not_received\",\"date_time\":\"2024-08-06T10:03:36.000Z\",\"dest_ip\":\"10.30.4.56\",\"dest_port\":\"443\",\"device_id\":\"N/A\",\"fragment\":\"\",\"geo_location\":\"N/A\",\"hostname\":\"f5qa\",\"http_class_name\":\"/Common/asmpolicy_lampqa\",\"ip_address_intelligence\":\"N/A\",\"ip_client\":\"10.43.24.23\",\"management_ip_address\":\"10.52.34.33\",\"management_ip_address_2\":\"N/A\",\"method\":\"HEAD\",\"microservice\":\"N/A\",\"originalRawData\":\"<134>Aug 6 12:03:36 f5qa ASM:unit_hostname=\\\"f5qa\\\",management_ip_address=\\\"10.52.34.33\\\",management_ip_address_2=\\\"N/A\\\",http_class_name=\\\"/Common/asmpolicy_lampqa\\\",web_application_name=\\\"/Common/asmpolicy_lampqa\\\",policy_name=\\\"/Common/asmpolicy_lampqa\\\",policy_apply_date=\\\"2024-07-02 16:43:55\\\",violations=\\\"N/A\\\",support_id=\\\"5410866668007843666\\\",request_status=\\\"passed\\\",response_code=\\\"200\\\",ip_client=\\\"10.43.24.23\\\",route_domain=\\\"0\\\",method=\\\"HEAD\\\",protocol=\\\"HTTPS\\\",query_string=\\\"\\\",x_forwarded_for_header_value=\\\"10.43.24.23, 10.43.24.23\\\",sig_ids=\\\"N/A\\\",sig_names=\\\"N/A\\\",date_time=\\\"2024-08-06 12:03:36\\\",severity=\\\"Informational\\\",attack_type=\\\"N/A\\\",geo_location=\\\"N/A\\\",ip_address_intelligence=\\\"N/A\\\",username=\\\"N/A\\\",session_id=\\\"7a60af492530220b\\\",src_port=\\\"50668\\\",dest_port=\\\"443\\\",dest_ip=\\\"10.30.4.56\\\",sub_violations=\\\"N/A\\\",virus_name=\\\"N/A\\\",violation_rating=\\\"0\\\",websocket_direction=\\\"N/A\\\",websocket_message_type=\\\"N/A\\\",device_id=\\\"N/A\\\",staged_sig_ids=\\\"\\\",staged_sig_names=\\\"\\\",threat_campaign_names=\\\"N/A\\\",staged_threat_campaign_names=\\\"N/A\\\",blocking_exception_reason=\\\"N/A\\\",captcha_result=\\\"not_received\\\",microservice=\\\"N/A\\\",tap_event_id=\\\"N/A\\\",tap_vid=\\\"N/A\\\",vs_name=\\\"/Common/vs_externalqa13_443\\\",sig_cves=\\\"N/A\\\",staged_sig_cves=\\\"N/A\\\",uri=\\\"/repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar\\\",fragment=\\\"\\\",request=\\\"HEAD /repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar HTTP/1.1\\\\r\\\\nCache-Control: no-cache, no-store\\\\r\\\\nPragma: no-cache\\\\r\\\\nHost: domain.gent\\\\r\\\\nConnection: Keep-Alive\\\\r\\\\nUser-Agent: Apache-Maven/3.9.7 (Java 17.0.12; Windows 11 10.0)\\\\r\\\\nAccept-Encoding: gzip,deflate\\\\r\\\\nAuthorization: Basic dc2VydmljZWZhY3Rvcnk6TE1YaHZwRUxhRjJodEFScWFQkkk=\\\\r\\\\nX-Forwarded-For: 10.43.24.23, 10.43.24.23\\\\r\\\\nX-Forwarded-Proto: https\\\\r\\\\n\\\\r\\\\n\\\",response=\\\"Response logging disabled\\\"\",\"policy_apply_date\":\"2024-07-02 16:43:55\",\"policy_name\":\"/Common/asmpolicy_lampqa\",\"protocol\":\"HTTPS\",\"query_string\":\"\",\"request\":\"HEAD /repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar HTTP/1.1\\\\r\\\\nCache-Control: no-cache, no-store\\\\r\\\\nPragma: no-cache\\\\r\\\\nHost: domain.gent\\\\r\\\\nConnection: Keep-Alive\\\\r\\\\nUser-Agent: Apache-Maven/3.9.7 (Java 17.0.12; Windows 11 10.0)\\\\r\\\\nAccept-Encoding: gzip,deflate\\\\r\\\\nAuthorization: Basic dc2VydmljZWZhY3Rvcnk6TE1YaHZwRUxhRjJodEFScWFQkkk=\\\\r\\\\nX-Forwarded-For: 10.43.24.23, 10.43.24.24\\\\r\\\\nX-Forwarded-Proto: https\\\\r\\\\n\\\\r\\\\n\",\"request_status\":\"passed\",\"response\":\"Response logging disabled\",\"response_code\":\"200\",\"route_domain\":\"0\",\"session_id\":\"7a60af492530220b\",\"severity\":\"Informational\",\"sig_cves\":\"N/A\",\"sig_ids\":\"N/A\",\"sig_names\":\"N/A\",\"src_port\":\"50668\",\"staged_sig_cves\":\"N/A\",\"staged_sig_ids\":\"\",\"staged_sig_names\":\"\",\"staged_threat_campaign_names\":\"N/A\",\"sub_violations\":\"N/A\",\"support_id\":\"5410868666607846666\",\"tap_event_id\":\"N/A\",\"tap_vid\":\"N/A\",\"telemetryEventCategory\":\"ASM\",\"tenant\":\"Common\",\"threat_campaign_names\":\"N/A\",\"uri\":\"/repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar\",\"username\":\"N/A\",\"violation_rating\":\"0\",\"violations\":\"N/A\",\"virus_name\":\"N/A\",\"vs_name\":\"/Common/vs_externalqa13_443\",\"web_application_name\":\"/Common/asmpolicy_lampqa\",\"websocket_direction\":\"N/A\",\"websocket_message_type\":\"N/A\",\"x_forwarded_for_header_value\":\"10.43.24.23, 10.43.24.24\"}", + "type": [ + "info" + ] + }, + "f5_bigip": { + "log": { + "captcha_result": "not_received", + "client": { + "ip": "10.43.24.23" + }, + "date_time": "2024-08-06T10:03:36.000Z", + "dest": { + "ip": "10.30.4.56", + "port": 443 + }, + "hostname": "f5qa", + "http": { + "class_name": "/Common/asmpolicy_lampqa" + }, + "management": { + "ip_address": "10.52.34.33" + }, + "method": "HEAD", + "policy": { + "apply_date": "2024-07-02T16:43:55.000Z", + "name": "/Common/asmpolicy_lampqa" + }, + "protocol": "HTTPS", + "request": { + "detail": "HEAD /repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar HTTP/1.1\\r\\nCache-Control: no-cache, no-store\\r\\nPragma: no-cache\\r\\nHost: domain.gent\\r\\nConnection: Keep-Alive\\r\\nUser-Agent: Apache-Maven/3.9.7 (Java 17.0.12; Windows 11 10.0)\\r\\nAccept-Encoding: gzip,deflate\\r\\nAuthorization: Basic dc2VydmljZWZhY3Rvcnk6TE1YaHZwRUxhRjJodEFScWFQkkk=\\r\\nX-Forwarded-For: 10.43.24.23, 10.43.24.24\\r\\nX-Forwarded-Proto: https\\r\\n\\r\\n", + "status": "passed" + }, + "response": { + "code": 200, + "value": "Response logging disabled" + }, + "route_domain": "0", + "session": { + "id": "7a60af492530220b" + }, + "severity": { + "name": "Informational" + }, + "src": { + "port": 50668 + }, + "support": { + "id": "5410868666607846666" + }, + "telemetry": { + "event": { + "category": "ASM" + } + }, + "tenant": "Common", + "uri": "/repository/maven-public/io/netty/netty-codec-haproxy/4.1.110.Final/netty-codec-haproxy-4.1.110.Final.jar", + "violation": { + "rating": 0 + }, + "vs_name": "/Common/vs_externalqa13_443", + "web_application_name": "/Common/asmpolicy_lampqa", + "x_forwarded_for_header_value": [ + "10.43.24.23", + "10.43.24.24" + ] + } + }, + "host": { + "name": "f5qa" + }, + "http": { + "request": { + "method": "HEAD" + } + }, + "log": { + "level": "informational" + }, + "network": { + "protocol": "https" + }, + "observer": { + "product": "Application Security Module", + "vendor": "F5" + }, + "related": { + "hosts": [ + "f5qa" + ], + "ip": [ + "10.43.24.23", + "10.30.4.56", + "10.52.34.33", + "10.43.24.24" + ] + }, + "server": { + "ip": "10.30.4.56", + "port": 443 + }, + "source": { + "ip": "10.43.24.23", + "port": 50668 + }, + "tags": [ + "preserve_original_event", + "preserve_duplicate_custom_fields" + ] } ] } \ No newline at end of file diff --git a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipasm.yml b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipasm.yml index f9950563585..cdaf832b13a 100644 --- a/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipasm.yml +++ b/packages/f5_bigip/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bigipasm.yml @@ -521,6 +521,10 @@ processors: tag: rename_websocket_message_type target_field: f5_bigip.log.websocket.message_type ignore_missing: true + - split: + field: json.x_forwarded_for_header_value + separator: ',\s*' + ignore_missing: true - convert: field: json.x_forwarded_for_header_value tag: convert_x_forwarded_for_header_value_to_ip @@ -532,12 +536,16 @@ processors: - append: field: error.message value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - - append: - field: related.ip - tag: append_related_ip - value: '{{{f5_bigip.log.x_forwarded_for_header_value}}}' - allow_duplicates: false - ignore_failure: true + - foreach: + field: f5_bigip.log.x_forwarded_for_header_value + if: ctx.f5_bigip?.log?.x_forwarded_for_header_value instanceof List + processor: + append: + field: related.ip + tag: append_related_ip + value: '{{{_ingest._value}}}' + allow_duplicates: false + ignore_failure: true - rename: field: json.geo_info tag: rename_geo_info diff --git a/packages/f5_bigip/data_stream/log/sample_event.json b/packages/f5_bigip/data_stream/log/sample_event.json index e2f3be22774..79cc2a7b22b 100644 --- a/packages/f5_bigip/data_stream/log/sample_event.json +++ b/packages/f5_bigip/data_stream/log/sample_event.json @@ -1,11 +1,11 @@ { "@timestamp": "2018-11-19T22:34:40.000Z", "agent": { - "ephemeral_id": "e5f59545-d8ac-4f69-9699-79bb945dff15", - "id": "5dfb7c6f-2d06-40bd-9835-16a8fd432357", + "ephemeral_id": "5783a7ca-031c-49a4-a74c-6bf741bd44a7", + "id": "7036eed3-e508-4a47-99a4-b144655eb291", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.13.0" + "version": "8.14.0" }, "client": { "ip": "81.2.69.142", @@ -13,7 +13,7 @@ }, "data_stream": { "dataset": "f5_bigip.log", - "namespace": "25415", + "namespace": "45148", "type": "logs" }, "destination": { @@ -24,9 +24,9 @@ "version": "8.11.0" }, "elastic_agent": { - "id": "5dfb7c6f-2d06-40bd-9835-16a8fd432357", + "id": "7036eed3-e508-4a47-99a4-b144655eb291", "snapshot": false, - "version": "8.13.0" + "version": "8.14.0" }, "event": { "agent_id_status": "verified", @@ -34,7 +34,7 @@ "network" ], "dataset": "f5_bigip.log", - "ingested": "2024-07-19T11:02:41Z", + "ingested": "2024-08-06T23:25:39Z", "kind": "alert", "original": "{\"application\":\"app.app\",\"attack_type\":\"Detection Evasion\",\"blocking_exception_reason\":\"test\",\"captcha_result\":\"not_received\",\"date_time\":\"2018-11-19 22:34:40\",\"dest_ip\":\"81.2.69.142\",\"dest_port\":\"80\",\"device_id\":\"12bdca32\",\"fragment\":\"test_Fragment\",\"geo_location\":\"US\",\"hostname\":\"hostname\",\"http_class_name\":\"/Common/abc/test\",\"ip_address_intelligence\":\"host1\",\"ip_client\":\"81.2.69.142\",\"management_ip_address\":\"81.2.69.142\",\"management_ip_address_2\":\"81.2.69.144\",\"method\":\"GET\",\"policy_apply_date\":\"2018-11-19 22:17:57\",\"policy_name\":\"/Common/abc\",\"protocol\":\"HTTP\",\"query_string\":\"name=abc\",\"request\":\"GET /admin/.\",\"request_status\":\"blocked\",\"response_code\":\"0\",\"route_domain\":\"example.com\",\"session_id\":\"abc123abcd\",\"severity\":\"Critical\",\"sig_ids\":\"abc12bcd\",\"sig_names\":\"Sig_Name\",\"src_port\":\"49804\",\"staged_sig_ids\":\"abc23121bc\",\"staged_sig_names\":\"test_name\",\"staged_threat_campaign_names\":\"test\",\"sub_violations\":\"Evasion technique detected:Directory traversals\",\"support_id\":\"123456789\",\"telemetryEventCategory\":\"ASM\",\"tenant\":\"Common\",\"threat_campaign_names\":\"threat\",\"uri\":\"/directory/file\",\"username\":\"test User\",\"violation_rating\":\"3\",\"violations\":\"Evasion technique detected\",\"virus_name\":\"test Virus\",\"web_application_name\":\"/Common/abc\",\"websocket_direction\":\"test\",\"websocket_message_type\":\"test\",\"x_forwarded_for_header_value\":\"81.2.69.144\"}", "type": [ @@ -135,7 +135,9 @@ "direction": "test", "message_type": "test" }, - "x_forwarded_for_header_value": "81.2.69.144" + "x_forwarded_for_header_value": [ + "81.2.69.144" + ] } }, "host": { diff --git a/packages/f5_bigip/docs/README.md b/packages/f5_bigip/docs/README.md index fc75332656c..9710b00e474 100644 --- a/packages/f5_bigip/docs/README.md +++ b/packages/f5_bigip/docs/README.md @@ -164,11 +164,11 @@ An example event for `log` looks as following: { "@timestamp": "2018-11-19T22:34:40.000Z", "agent": { - "ephemeral_id": "e5f59545-d8ac-4f69-9699-79bb945dff15", - "id": "5dfb7c6f-2d06-40bd-9835-16a8fd432357", + "ephemeral_id": "5783a7ca-031c-49a4-a74c-6bf741bd44a7", + "id": "7036eed3-e508-4a47-99a4-b144655eb291", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.13.0" + "version": "8.14.0" }, "client": { "ip": "81.2.69.142", @@ -176,7 +176,7 @@ An example event for `log` looks as following: }, "data_stream": { "dataset": "f5_bigip.log", - "namespace": "25415", + "namespace": "45148", "type": "logs" }, "destination": { @@ -187,9 +187,9 @@ An example event for `log` looks as following: "version": "8.11.0" }, "elastic_agent": { - "id": "5dfb7c6f-2d06-40bd-9835-16a8fd432357", + "id": "7036eed3-e508-4a47-99a4-b144655eb291", "snapshot": false, - "version": "8.13.0" + "version": "8.14.0" }, "event": { "agent_id_status": "verified", @@ -197,7 +197,7 @@ An example event for `log` looks as following: "network" ], "dataset": "f5_bigip.log", - "ingested": "2024-07-19T11:02:41Z", + "ingested": "2024-08-06T23:25:39Z", "kind": "alert", "original": "{\"application\":\"app.app\",\"attack_type\":\"Detection Evasion\",\"blocking_exception_reason\":\"test\",\"captcha_result\":\"not_received\",\"date_time\":\"2018-11-19 22:34:40\",\"dest_ip\":\"81.2.69.142\",\"dest_port\":\"80\",\"device_id\":\"12bdca32\",\"fragment\":\"test_Fragment\",\"geo_location\":\"US\",\"hostname\":\"hostname\",\"http_class_name\":\"/Common/abc/test\",\"ip_address_intelligence\":\"host1\",\"ip_client\":\"81.2.69.142\",\"management_ip_address\":\"81.2.69.142\",\"management_ip_address_2\":\"81.2.69.144\",\"method\":\"GET\",\"policy_apply_date\":\"2018-11-19 22:17:57\",\"policy_name\":\"/Common/abc\",\"protocol\":\"HTTP\",\"query_string\":\"name=abc\",\"request\":\"GET /admin/.\",\"request_status\":\"blocked\",\"response_code\":\"0\",\"route_domain\":\"example.com\",\"session_id\":\"abc123abcd\",\"severity\":\"Critical\",\"sig_ids\":\"abc12bcd\",\"sig_names\":\"Sig_Name\",\"src_port\":\"49804\",\"staged_sig_ids\":\"abc23121bc\",\"staged_sig_names\":\"test_name\",\"staged_threat_campaign_names\":\"test\",\"sub_violations\":\"Evasion technique detected:Directory traversals\",\"support_id\":\"123456789\",\"telemetryEventCategory\":\"ASM\",\"tenant\":\"Common\",\"threat_campaign_names\":\"threat\",\"uri\":\"/directory/file\",\"username\":\"test User\",\"violation_rating\":\"3\",\"violations\":\"Evasion technique detected\",\"virus_name\":\"test Virus\",\"web_application_name\":\"/Common/abc\",\"websocket_direction\":\"test\",\"websocket_message_type\":\"test\",\"x_forwarded_for_header_value\":\"81.2.69.144\"}", "type": [ @@ -298,7 +298,9 @@ An example event for `log` looks as following: "direction": "test", "message_type": "test" }, - "x_forwarded_for_header_value": "81.2.69.144" + "x_forwarded_for_header_value": [ + "81.2.69.144" + ] } }, "host": { diff --git a/packages/f5_bigip/manifest.yml b/packages/f5_bigip/manifest.yml index aee4b657014..cbea470bfe4 100644 --- a/packages/f5_bigip/manifest.yml +++ b/packages/f5_bigip/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: f5_bigip title: F5 BIG-IP -version: "1.18.1" +version: "1.19.0" description: Collect logs from F5 BIG-IP with Elastic Agent. type: integration categories: From 7b5d67c96d2270c65ca5db4b7532fde1683d116a Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 8 Aug 2024 15:35:05 +0200 Subject: [PATCH 11/13] [zscaler_zia] Update response format version numbers (#10741) Response formats were often updated without incrementing the corresponding version number. These have been updated to reflect the number of changes made to the response formats, as summarized in the following table. | Data stream | Was | Now | Note | |------------------|-----|-----|-------------------------------------| | Alerts | v1 | v1 | Correct. | | Audit Log | v1 | v1 | Correct. Pattern added in 78f3eae. | | DNS Log | v1 | v2 | Fixed. Pattern updated in 78f3eae. | | Endpoint DLP Log | v1 | v1 | Correct. Pattern added in 78f3eae. | | Firewall Log | v1 | v2 | Fixed. Pattern updated in 78f3eae. | | Tunnel Log | v1 | v2 | Fixed. Patterns updated in 78f3eae. | | Web Log | v2 | v5 | Fixed. Pattern updated in... | | | | | 3418fe2, a9783c8, 78f3eae, dbf3f74. | --- packages/zscaler_zia/_dev/build/docs/README.md | 14 +++++++------- packages/zscaler_zia/changelog.yml | 5 +++++ packages/zscaler_zia/docs/README.md | 14 +++++++------- packages/zscaler_zia/manifest.yml | 2 +- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/packages/zscaler_zia/_dev/build/docs/README.md b/packages/zscaler_zia/_dev/build/docs/README.md index c02d836f105..5986d3fb15b 100644 --- a/packages/zscaler_zia/_dev/build/docs/README.md +++ b/packages/zscaler_zia/_dev/build/docs/README.md @@ -98,7 +98,7 @@ Note: Please make sure to use latest version of given response formats. See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/about-alerts) -Zscaler response format (v1): +Zscaler Alerts response format (v1): ``` <%d{syslogid}>%s{Monthname} %2d{Dayofmonth} %02d{Hour}:%02d{Minutes}:%02d{Seconds} [%s{Deviceip}] ZscalerNSS: %s{Eventinfo}\n ``` @@ -114,7 +114,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/adding-cloud-nss-feeds-admin-audit-logs) -Zscaler response format (v1): +Zscaler Audit Log response format (v1): ``` \{"sourcetype":"zscalernss-audit","event":\{"time":"%s{time}","recordid":"%d{recordid}","action":"%s{action}","category":"%s{category}","subcategory":"%s{subcategory}","resource":"%s{resource}","interface":"%s{interface}","adminid":"%s{adminid}","clientip":"%s{clientip}","result":"%s{result}","errorcode":"%s{errorcode}","auditlogtype":"%s{auditlogtype}","preaction":%s{preaction},"postaction":%s{postaction}\}\} ``` @@ -131,7 +131,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-dns-logs) -Zscaler response format (v1): +Zscaler DNS Log response format (v2): ``` \{"sourcetype":"zscalernss-dns","event":\{"user":"%s{elogin}","department":"%s{edepartment}","location":"%s{elocation}","clt_sip":"%s{cip}","cloudname":"%s{cloudname}","company":"%s{company}","datacenter":"%s{datacenter}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","day_of_month":"%02d{dd}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","devicehostname":"%s{devicehostname}","devicemodel":"%s{devicemodel}","devicename":"%s{devicename}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","deviceowner":"%s{deviceowner}","devicetype":"%s{devicetype}","dnsapp":"%s{dnsapp}","dnsappcat":"%s{dnsappcat}","dns_gateway_status":"%s{dnsgw_flags}","dns_gateway_rule":"%s{dnsgw_slot}","dns_gateway_server_protocol":"%s{dnsgw_srv_proto}","category":"%s{domcat}","durationms":"%d{durationms}","ecs_prefix":"%s{ecs_prefix}","ecs_slot":"%s{ecs_slot}","epochtime":"%d{epochtime}","error":"%s{error}","hour":"%02d{hh}","http_code":"%s{http_code}","istcp":"%d{istcp}","loc":"%s{location}","login":"%s{login}","minutes":"%02d{mm}","month":"%s{mon}","month_of_year":"%02d{mth}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odomcat":"%s{odomcat}","protocol":"%s{protocol}","recordid":"%d{recordid}","dns_req":"%s{req}","reqaction":"%s{reqaction}","reqrulelabel":"%s{reqrulelabel}","dns_reqtype":"%s{reqtype}","dns_resp":"%s{res}","resaction":"%s{resaction}","respipcategory":"%s{respipcat}","resrulelabel":"%s{resrulelabel}","restype":"%s{restype}","srv_dip":"%s{sip}","srv_dport":"%d{sport}","second":"%02d{ss}","datetime":"%s{time}","tz":"%s{tz}","year":"%04d{yyyy}"\}\} ``` @@ -148,7 +148,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-endpoint-dlp-logs) -Zscaler response format (v1): +Zscaler Endpoint DLP Log response format (v1): ``` \{"sourcetype":"zscalernss-edlp","event":\{"actiontaken":"%s{actiontaken}","activitytype":"%s{activitytype}","additionalinfo":"%s{addinfo}","channel":"%s{channel}","confirmaction":"%s{confirmaction}","confirmjustification":"%s{confirmjust}","datacenter":"%s{datacenter}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","day":"%s{day}","dd":"%02d{dd}","department":"%s{department}","deviceappversion":"%s{deviceappversion}","devicehostname":"%s{devicehostname}","devicemodel":"%s{devicemodel}","devicename":"%s{devicename}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","deviceowner":"%s{deviceowner}","deviceplatform":"%s{deviceplatform}","devicetype":"%s{devicetype}","dlpdictcount":"%s{dlpcounts}","dlpdictnames":"%s{dlpdictnames}","dlpenginenames":"%s{dlpengnames}","dlpidentifier":"%llu{dlpidentifier}","dsttype":"%s{dsttype}","eventtime":"%s{eventtime}","expectedaction":"%s{expectedaction}","filedoctype":"%s{filedoctype}","filedstpath":"%s{filedstpath}","filemd5":"%s{filemd5}","filesha":"%s{filesha}","filesrcpath":"%s{filesrcpath}","filetypecategory":"%s{filetypecategory}","filetypename":"%s{filetypename}","hh":"%02d{hh}","itemdstname":"%s{itemdstname}","itemname":"%s{itemname}","itemsrcname":"%s{itemsrcname}","itemtype":"%s{itemtype}","logtype":"%s{logtype}","mm":"%02d{mm}","mon":"%s{mon}","mth":"%02d{mth}","numdlpdictids":"%u{numdlpdictids}","numdlpengineids":"%u{numdlpengids}","odepartment":"%s{odepartment}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odlpdictnames":"%s{odlpdictnames}","odlpenginenames":"%s{odlpengnames}","ofiledstpath":"%s{ofiledstpath}","ofilesrcpath":"%s{ofilesrcpath}","oitemdstname":"%s{oitemdstname}","oitemname":"%s{oitemname}","oitemsrcname":"%s{oitemsrcname}","ootherrulelabels":"%s{ootherrulelabels}","otherrulelabels":"%s{otherrulelabels}","orulename":"%s{otriggeredrulelabel}","ouser":"%s{ouser}","recordid":"%llu{recordid}","feedtime":"%s{rtime}","scannedbytes":"%llu{scanned_bytes}","scantime":"%llu{scantime}","severity":"%s{severity}","srctype":"%s{srctype}","ss":"%02d{ss}","datetime":"%s{time}","rulename":"%s{triggeredrulelabel}","timezone":"%s{tz}","user":"%s{user}","yyyy":"%04d{yyyy}","zdpmode":"%s{zdpmode}"\}\} ``` @@ -165,7 +165,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-firewall-logs) -Zscaler response format (v1): +Zscaler Firewall Log response format (v2): ``` \{"sourcetype":"zscalernss-fw","event":\{"datetime":"%s{time}","outbytes":"%ld{outbytes}","cltdomain":"%s{cdfqdn}","destcountry":"%s{destcountry}","cdip":"%s{cdip}","sdip":"%s{sdip}","cdport":"%d{cdport}","sdport":"%d{sdport}","devicemodel":"%s{devicemodel}","action":"%s{action}","duration":"%d{duration}","recordid":"%d{recordid}","tz":"%s{tz}","devicename":"%s{devicename}","devicehostname":"%s{devicehostname}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","nwapp":"%s{nwapp}","nwsvc":"%s{nwsvc}","proto":"%s{ipproto}","ipsrulelabel":"%s{ipsrulelabel}","dnatrulelabel":"%s{dnatrulelabel}","rdr_rulename":"%s{rdr_rulename}","rule":"%s{rulelabel}","rulelabel":"%s{erulelabel}","inbytes":"%ld{inbytes}","srcipcountry":"%s{srcip_country}","csip":"%s{csip}","ssip":"%s{ssip}","csport":"%d{csport}","ssport":"%d{ssport}","user":"%s{elogin}","aggregate":"%s{aggregate}","bypassed_session":"%d{bypassed_session}","bypass_time":"%s{bypass_etime}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","datacenter":"%s{datacenter}","day_of_month":"%02d{dd}","department":"%s{edepartment}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","deviceowner":"%s{deviceowner}","avgduration":"%d{avgduration}","durationms":"%d{durationms}","epochtime":"%d{epochtime}","external_deviceid":"%s{external_deviceid}","flow_type":"%s{flow_type}","forward_gateway_name":"%s{fwd_gw_name}","hour":"%02d{hh}","ipcat":"%s{ipcat}","ips_custom_signature":"%d{ips_custom_signature}","location":"%s{location}","locationname":"%s{elocation}","login":"%s{login}","minute":"%02d{mm}","month":"%s{mon}","month_of_year":"%02d{mth}","dnat":"%s{dnat}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","ofwd_gw_name":"%s{ofwd_gw_name}","odevicehostname":"%s{odevicehostname}","oipcat":"%s{oipcat}","oipsrulelabel":"%s{oipsrulelabel}","ordr_rulename":"%s{ordr_rulename}","orulelabel":"%s{orulelabel}","ozpa_app_seg_name":"%s{ozpa_app_seg_name}","second":"%02d{ss}","numsessions":"%d{numsessions}","stateful":"%s{stateful}","threat_name":"%s{threatname}","threatcat":"%s{threatcat}","threatname":"%s{ethreatname}","tsip":"%s{tsip}","tuntype":"%s{ttype}","year":"%04d{yyyy}","ztunnelversion":"%s{ztunnelversion}","zpa_app_seg_name":"%s{zpa_app_seg_name}"\}\} ``` @@ -182,7 +182,7 @@ Sample Response: See: [Zscaler Vendor documentation]( https://help.zscaler.com/zia/nss-feed-output-format-tunnel-logs) -Zscaler response format (v1): +Zscaler Tunnel Log response formats (v2): - Tunnel Event: ``` \{"sourcetype":"zscalernss-tunnel","event":\{"datetime":"%s{datetime}","day":"%s{day}","dd":"%02d{dd}","destinationip":"%s{destvip}","event":"%s{event}","eventreason":"%s{eventreason}","hh":"%02d{hh}","locationname":"%s{locationname}","mm":"%02d{mm}","mon":"%s{mon}","mth":"%02d{mth}","olocationname":"%s{olocationname}","ovpncredentialname":"%s{ovpncredentialname}","recordid":"%d{recordid}","sourceip":"%s{sourceip}","sourceport":"%d{srcport}","ss":"%02d{ss}","Recordtype":"%s{tunnelactionname}","tunneltype":"%s{tunneltype}","timezone":"%s{tz}","user":"%s{vpncredentialname}","yyyy":"%04d{yyyy}"\}\} @@ -214,7 +214,7 @@ Sample Response: ![Escape feed setup image](../img/escape_feed.png?raw=true) See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-web-logs) -Zscaler response format (v2): +Zscaler Web Log response format (v5): ``` \{"sourcetype":"zscalernss-web","event":\{"time":"%s{time}","cloudname":"%s{cloudname}","host":"%s{host}","serverip":"%s{sip}","external_devid":"%s{external_devid}","devicemodel":"%s{devicemodel}","action":"%s{action}","recordid":"%d{recordid}","reason":"%s{reason}","threatseverity":"%s{threatseverity}","tz":"%s{tz}","filesubtype":"%s{filesubtype}","upload_filesubtype":"%s{upload_filesubtype}","sha256":"%s{sha256}","bamd5":"%s{bamd5}","filename":"%s{filename}","upload_filename":"%s{upload_filename}","filetype":"%s{filetype}","devicename":"%s{devicename}","devicehostname":"%s{devicehostname}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","devicetype":"%s{devicetype}","reqsize":"%d{reqsize}","reqmethod":"%s{reqmethod}","refererurl":"%s{referer}","respsize":"%d{respsize}","respcode":"%s{respcode}","reqversion":"%s{reqversion}","respversion":"%s{respversion}","proto":"%s{proto}","company":"%s{company}","dlpmd5":"%s{dlpmd5}","apprulelabel":"%s{apprulelabel}","dlprulename":"%s{dlprulename}","rulelabel":"%s{rulelabel}","urlfilterrulelabel":"%s{urlfilterrulelabel}","cltip":"%s{cip}","cltintip":"%s{cintip}","cltsourceport":"%d{clt_sport}","threatname":"%s{threatname}","cltsslcipher":"%s{clientsslcipher}","clttlsversion":"%s{clienttlsversion}","eurl":"%s{eurl}","url":"%s{url}","useragent":"%s{ua}","login":"%s{login}","applayerprotocol":"%s{alpnprotocol}","appclass":"%s{appclass}","appname":"%s{appname}","appriskscore":"%s{app_risk_score}","bandwidthclassname":"%s{bwclassname}","bandwidthrulename":"%s{bwrulename}","bwthrottle":"%s{bwthrottle}","bypassedtime":"%s{bypassed_etime}","bypassedtraffic":"%d{bypassed_traffic}","cltsslsessreuse":"%s{clientsslsessreuse}","cltpubip":"%s{cpubip}","cltsslfailcount":"%d{cltsslfailcount}","cltsslfailreason":"%s{cltsslfailreason}","contenttype":"%s{contenttype}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","datacenter":"%s{datacenter}","day":"%s{day}","day_of_month":"%02d{dd}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","deviceowner":"%s{deviceowner}","df_hosthead":"%s{df_hosthead}","df_hostname":"%s{df_hostname}","dlpdicthitcount":"%s{dlpdicthitcount}","dlpdict":"%s{dlpdict}","dlpeng":"%s{dlpeng}","dlpidentifier":"%d{dlpidentifier}","eedone":"%s{eedone}","epochtime":"%d{epochtime}","fileclass":"%s{fileclass}","flow_type":"%s{flow_type}","forward_gateway_ip":"%s{fwd_gw_ip}","forward_gateway_name":"%s{fwd_gw_name}","forward_type":"%s{fwd_type}","hour":"%02d{hh}","is_sslexpiredca":"%s{is_sslexpiredca}","is_sslselfsigned":"%s{is_sslselfsigned}","is_ssluntrustedca":"%s{is_ssluntrustedca}","keyprotectiontype":"%s{keyprotectiontype}","location":"%s{location}","malwarecategory":"%s{malwarecat}","malwareclass":"%s{malwareclass}","minute":"%02d{mm}","mobappcategory":"%s{mobappcat}","mobappname":"%s{mobappname}","mobdevtype":"%s{mobdevtype}","module":"%s{module}","month":"%s{mon}","month_of_year":"%02d{mth}","nssserviceip":"%s{nsssvcip}","oapprulelabel":"%s{oapprulelabel}","obwclassname":"%s{obwclassname}","ocip":"%d{ocip}","ocpubip":"%d{ocpubip}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odlpdict":"%s{odlpdict}","odlpeng":"%s{odlpeng}","odlprulename":"%s{odlprulename}","ofwd_gw_name":"%s{ofwd_gw_name}","ologin":"%s{ologin}","ordr_rulename":"%s{ordr_rulename}","ourlcat":"%s{ourlcat}","ourlfilterrulelabel":"%s{ourlfilterrulelabel}","ozpa_app_seg_name":"%s{ozpa_app_seg_name}","externalsslpolicyreason":"%s{externalspr}","productversion":"%s{productversion}","rdr_rulename":"%s{rdr_rulename}","refererhost":"%s{refererhost}","reqheadersize":"%d{reqhdrsize}","reqdatasize":"%d{reqdatasize}","respheadersize":"%d{resphdrsize}","respdatasize":"%d{respdatasize}","riskscore":"%d{riskscore}","ruletype":"%s{ruletype}","second":"%02d{ss}","srvcertchainvalpass":"%s{srvcertchainvalpass}","srvcertvalidationtype":"%s{srvcertvalidationtype}","srvcertvalidityperiod":"%s{srvcertvalidityperiod}","srvsslcipher":"%s{srvsslcipher}","serversslsessreuse":"%s{serversslsessreuse}","srvocspresult":"%s{srvocspresult}","srvtlsversion":"%s{srvtlsversion}","srvwildcardcert":"%s{srvwildcardcert}","ssldecrypted":"%s{ssldecrypted}","throttlereqsize":"%d{throttlereqsize}","throttlerespsize":"%d{throttlerespsize}","totalsize":"%d{totalsize}","trafficredirectmethod":"%s{trafficredirectmethod}","unscannabletype":"%s{unscannabletype}","upload_doctypename":"%s{upload_doctypename}","upload_fileclass":"%s{upload_fileclass}","upload_filetype":"%s{upload_filetype}","urlcatmethod":"%s{urlcatmethod}","urlsubcat":"%s{urlcat}","urlsupercat":"%s{urlsupercat}","urlclass":"%s{urlclass}","useragentclass":"%s{uaclass}","useragenttoken":"%s{ua_token}","userlocationname":"%s{userlocationname}","year":"%04d{yyyy}","ztunnelversion":"%s{ztunnelversion}","zpa_app_seg_name":"%s{zpa_app_seg_name}"\}\} ``` diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml index 9f5a8b65cf1..3eda3016822 100644 --- a/packages/zscaler_zia/changelog.yml +++ b/packages/zscaler_zia/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "3.0.4" + changes: + - description: Update response format version numbers. + type: bugfix + link: https://github.com/elastic/integrations/pull/10741 - version: "3.0.3" changes: - description: Add eurl hex-encoded field for url parsing. diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md index fb634ef8471..ed24eeb719d 100644 --- a/packages/zscaler_zia/docs/README.md +++ b/packages/zscaler_zia/docs/README.md @@ -98,7 +98,7 @@ Note: Please make sure to use latest version of given response formats. See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/about-alerts) -Zscaler response format (v1): +Zscaler Alerts response format (v1): ``` <%d{syslogid}>%s{Monthname} %2d{Dayofmonth} %02d{Hour}:%02d{Minutes}:%02d{Seconds} [%s{Deviceip}] ZscalerNSS: %s{Eventinfo}\n ``` @@ -114,7 +114,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/adding-cloud-nss-feeds-admin-audit-logs) -Zscaler response format (v1): +Zscaler Audit Log response format (v1): ``` \{"sourcetype":"zscalernss-audit","event":\{"time":"%s{time}","recordid":"%d{recordid}","action":"%s{action}","category":"%s{category}","subcategory":"%s{subcategory}","resource":"%s{resource}","interface":"%s{interface}","adminid":"%s{adminid}","clientip":"%s{clientip}","result":"%s{result}","errorcode":"%s{errorcode}","auditlogtype":"%s{auditlogtype}","preaction":%s{preaction},"postaction":%s{postaction}\}\} ``` @@ -131,7 +131,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-dns-logs) -Zscaler response format (v1): +Zscaler DNS Log response format (v2): ``` \{"sourcetype":"zscalernss-dns","event":\{"user":"%s{elogin}","department":"%s{edepartment}","location":"%s{elocation}","clt_sip":"%s{cip}","cloudname":"%s{cloudname}","company":"%s{company}","datacenter":"%s{datacenter}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","day_of_month":"%02d{dd}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","devicehostname":"%s{devicehostname}","devicemodel":"%s{devicemodel}","devicename":"%s{devicename}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","deviceowner":"%s{deviceowner}","devicetype":"%s{devicetype}","dnsapp":"%s{dnsapp}","dnsappcat":"%s{dnsappcat}","dns_gateway_status":"%s{dnsgw_flags}","dns_gateway_rule":"%s{dnsgw_slot}","dns_gateway_server_protocol":"%s{dnsgw_srv_proto}","category":"%s{domcat}","durationms":"%d{durationms}","ecs_prefix":"%s{ecs_prefix}","ecs_slot":"%s{ecs_slot}","epochtime":"%d{epochtime}","error":"%s{error}","hour":"%02d{hh}","http_code":"%s{http_code}","istcp":"%d{istcp}","loc":"%s{location}","login":"%s{login}","minutes":"%02d{mm}","month":"%s{mon}","month_of_year":"%02d{mth}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odomcat":"%s{odomcat}","protocol":"%s{protocol}","recordid":"%d{recordid}","dns_req":"%s{req}","reqaction":"%s{reqaction}","reqrulelabel":"%s{reqrulelabel}","dns_reqtype":"%s{reqtype}","dns_resp":"%s{res}","resaction":"%s{resaction}","respipcategory":"%s{respipcat}","resrulelabel":"%s{resrulelabel}","restype":"%s{restype}","srv_dip":"%s{sip}","srv_dport":"%d{sport}","second":"%02d{ss}","datetime":"%s{time}","tz":"%s{tz}","year":"%04d{yyyy}"\}\} ``` @@ -148,7 +148,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-endpoint-dlp-logs) -Zscaler response format (v1): +Zscaler Endpoint DLP Log response format (v1): ``` \{"sourcetype":"zscalernss-edlp","event":\{"actiontaken":"%s{actiontaken}","activitytype":"%s{activitytype}","additionalinfo":"%s{addinfo}","channel":"%s{channel}","confirmaction":"%s{confirmaction}","confirmjustification":"%s{confirmjust}","datacenter":"%s{datacenter}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","day":"%s{day}","dd":"%02d{dd}","department":"%s{department}","deviceappversion":"%s{deviceappversion}","devicehostname":"%s{devicehostname}","devicemodel":"%s{devicemodel}","devicename":"%s{devicename}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","deviceowner":"%s{deviceowner}","deviceplatform":"%s{deviceplatform}","devicetype":"%s{devicetype}","dlpdictcount":"%s{dlpcounts}","dlpdictnames":"%s{dlpdictnames}","dlpenginenames":"%s{dlpengnames}","dlpidentifier":"%llu{dlpidentifier}","dsttype":"%s{dsttype}","eventtime":"%s{eventtime}","expectedaction":"%s{expectedaction}","filedoctype":"%s{filedoctype}","filedstpath":"%s{filedstpath}","filemd5":"%s{filemd5}","filesha":"%s{filesha}","filesrcpath":"%s{filesrcpath}","filetypecategory":"%s{filetypecategory}","filetypename":"%s{filetypename}","hh":"%02d{hh}","itemdstname":"%s{itemdstname}","itemname":"%s{itemname}","itemsrcname":"%s{itemsrcname}","itemtype":"%s{itemtype}","logtype":"%s{logtype}","mm":"%02d{mm}","mon":"%s{mon}","mth":"%02d{mth}","numdlpdictids":"%u{numdlpdictids}","numdlpengineids":"%u{numdlpengids}","odepartment":"%s{odepartment}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odlpdictnames":"%s{odlpdictnames}","odlpenginenames":"%s{odlpengnames}","ofiledstpath":"%s{ofiledstpath}","ofilesrcpath":"%s{ofilesrcpath}","oitemdstname":"%s{oitemdstname}","oitemname":"%s{oitemname}","oitemsrcname":"%s{oitemsrcname}","ootherrulelabels":"%s{ootherrulelabels}","otherrulelabels":"%s{otherrulelabels}","orulename":"%s{otriggeredrulelabel}","ouser":"%s{ouser}","recordid":"%llu{recordid}","feedtime":"%s{rtime}","scannedbytes":"%llu{scanned_bytes}","scantime":"%llu{scantime}","severity":"%s{severity}","srctype":"%s{srctype}","ss":"%02d{ss}","datetime":"%s{time}","rulename":"%s{triggeredrulelabel}","timezone":"%s{tz}","user":"%s{user}","yyyy":"%04d{yyyy}","zdpmode":"%s{zdpmode}"\}\} ``` @@ -165,7 +165,7 @@ Sample Response: See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-firewall-logs) -Zscaler response format (v1): +Zscaler Firewall Log response format (v2): ``` \{"sourcetype":"zscalernss-fw","event":\{"datetime":"%s{time}","outbytes":"%ld{outbytes}","cltdomain":"%s{cdfqdn}","destcountry":"%s{destcountry}","cdip":"%s{cdip}","sdip":"%s{sdip}","cdport":"%d{cdport}","sdport":"%d{sdport}","devicemodel":"%s{devicemodel}","action":"%s{action}","duration":"%d{duration}","recordid":"%d{recordid}","tz":"%s{tz}","devicename":"%s{devicename}","devicehostname":"%s{devicehostname}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","nwapp":"%s{nwapp}","nwsvc":"%s{nwsvc}","proto":"%s{ipproto}","ipsrulelabel":"%s{ipsrulelabel}","dnatrulelabel":"%s{dnatrulelabel}","rdr_rulename":"%s{rdr_rulename}","rule":"%s{rulelabel}","rulelabel":"%s{erulelabel}","inbytes":"%ld{inbytes}","srcipcountry":"%s{srcip_country}","csip":"%s{csip}","ssip":"%s{ssip}","csport":"%d{csport}","ssport":"%d{ssport}","user":"%s{elogin}","aggregate":"%s{aggregate}","bypassed_session":"%d{bypassed_session}","bypass_time":"%s{bypass_etime}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","datacenter":"%s{datacenter}","day_of_month":"%02d{dd}","department":"%s{edepartment}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","deviceowner":"%s{deviceowner}","avgduration":"%d{avgduration}","durationms":"%d{durationms}","epochtime":"%d{epochtime}","external_deviceid":"%s{external_deviceid}","flow_type":"%s{flow_type}","forward_gateway_name":"%s{fwd_gw_name}","hour":"%02d{hh}","ipcat":"%s{ipcat}","ips_custom_signature":"%d{ips_custom_signature}","location":"%s{location}","locationname":"%s{elocation}","login":"%s{login}","minute":"%02d{mm}","month":"%s{mon}","month_of_year":"%02d{mth}","dnat":"%s{dnat}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","ofwd_gw_name":"%s{ofwd_gw_name}","odevicehostname":"%s{odevicehostname}","oipcat":"%s{oipcat}","oipsrulelabel":"%s{oipsrulelabel}","ordr_rulename":"%s{ordr_rulename}","orulelabel":"%s{orulelabel}","ozpa_app_seg_name":"%s{ozpa_app_seg_name}","second":"%02d{ss}","numsessions":"%d{numsessions}","stateful":"%s{stateful}","threat_name":"%s{threatname}","threatcat":"%s{threatcat}","threatname":"%s{ethreatname}","tsip":"%s{tsip}","tuntype":"%s{ttype}","year":"%04d{yyyy}","ztunnelversion":"%s{ztunnelversion}","zpa_app_seg_name":"%s{zpa_app_seg_name}"\}\} ``` @@ -182,7 +182,7 @@ Sample Response: See: [Zscaler Vendor documentation]( https://help.zscaler.com/zia/nss-feed-output-format-tunnel-logs) -Zscaler response format (v1): +Zscaler Tunnel Log response formats (v2): - Tunnel Event: ``` \{"sourcetype":"zscalernss-tunnel","event":\{"datetime":"%s{datetime}","day":"%s{day}","dd":"%02d{dd}","destinationip":"%s{destvip}","event":"%s{event}","eventreason":"%s{eventreason}","hh":"%02d{hh}","locationname":"%s{locationname}","mm":"%02d{mm}","mon":"%s{mon}","mth":"%02d{mth}","olocationname":"%s{olocationname}","ovpncredentialname":"%s{ovpncredentialname}","recordid":"%d{recordid}","sourceip":"%s{sourceip}","sourceport":"%d{srcport}","ss":"%02d{ss}","Recordtype":"%s{tunnelactionname}","tunneltype":"%s{tunneltype}","timezone":"%s{tz}","user":"%s{vpncredentialname}","yyyy":"%04d{yyyy}"\}\} @@ -214,7 +214,7 @@ Sample Response: ![Escape feed setup image](../img/escape_feed.png?raw=true) See: [Zscaler Vendor documentation](https://help.zscaler.com/zia/nss-feed-output-format-web-logs) -Zscaler response format (v2): +Zscaler Web Log response format (v5): ``` \{"sourcetype":"zscalernss-web","event":\{"time":"%s{time}","cloudname":"%s{cloudname}","host":"%s{host}","serverip":"%s{sip}","external_devid":"%s{external_devid}","devicemodel":"%s{devicemodel}","action":"%s{action}","recordid":"%d{recordid}","reason":"%s{reason}","threatseverity":"%s{threatseverity}","tz":"%s{tz}","filesubtype":"%s{filesubtype}","upload_filesubtype":"%s{upload_filesubtype}","sha256":"%s{sha256}","bamd5":"%s{bamd5}","filename":"%s{filename}","upload_filename":"%s{upload_filename}","filetype":"%s{filetype}","devicename":"%s{devicename}","devicehostname":"%s{devicehostname}","deviceostype":"%s{deviceostype}","deviceosversion":"%s{deviceosversion}","devicetype":"%s{devicetype}","reqsize":"%d{reqsize}","reqmethod":"%s{reqmethod}","refererurl":"%s{referer}","respsize":"%d{respsize}","respcode":"%s{respcode}","reqversion":"%s{reqversion}","respversion":"%s{respversion}","proto":"%s{proto}","company":"%s{company}","dlpmd5":"%s{dlpmd5}","apprulelabel":"%s{apprulelabel}","dlprulename":"%s{dlprulename}","rulelabel":"%s{rulelabel}","urlfilterrulelabel":"%s{urlfilterrulelabel}","cltip":"%s{cip}","cltintip":"%s{cintip}","cltsourceport":"%d{clt_sport}","threatname":"%s{threatname}","cltsslcipher":"%s{clientsslcipher}","clttlsversion":"%s{clienttlsversion}","eurl":"%s{eurl}","url":"%s{url}","useragent":"%s{ua}","login":"%s{login}","applayerprotocol":"%s{alpnprotocol}","appclass":"%s{appclass}","appname":"%s{appname}","appriskscore":"%s{app_risk_score}","bandwidthclassname":"%s{bwclassname}","bandwidthrulename":"%s{bwrulename}","bwthrottle":"%s{bwthrottle}","bypassedtime":"%s{bypassed_etime}","bypassedtraffic":"%d{bypassed_traffic}","cltsslsessreuse":"%s{clientsslsessreuse}","cltpubip":"%s{cpubip}","cltsslfailcount":"%d{cltsslfailcount}","cltsslfailreason":"%s{cltsslfailreason}","contenttype":"%s{contenttype}","datacentercity":"%s{datacentercity}","datacentercountry":"%s{datacentercountry}","datacenter":"%s{datacenter}","day":"%s{day}","day_of_month":"%02d{dd}","dept":"%s{dept}","deviceappversion":"%s{deviceappversion}","deviceowner":"%s{deviceowner}","df_hosthead":"%s{df_hosthead}","df_hostname":"%s{df_hostname}","dlpdicthitcount":"%s{dlpdicthitcount}","dlpdict":"%s{dlpdict}","dlpeng":"%s{dlpeng}","dlpidentifier":"%d{dlpidentifier}","eedone":"%s{eedone}","epochtime":"%d{epochtime}","fileclass":"%s{fileclass}","flow_type":"%s{flow_type}","forward_gateway_ip":"%s{fwd_gw_ip}","forward_gateway_name":"%s{fwd_gw_name}","forward_type":"%s{fwd_type}","hour":"%02d{hh}","is_sslexpiredca":"%s{is_sslexpiredca}","is_sslselfsigned":"%s{is_sslselfsigned}","is_ssluntrustedca":"%s{is_ssluntrustedca}","keyprotectiontype":"%s{keyprotectiontype}","location":"%s{location}","malwarecategory":"%s{malwarecat}","malwareclass":"%s{malwareclass}","minute":"%02d{mm}","mobappcategory":"%s{mobappcat}","mobappname":"%s{mobappname}","mobdevtype":"%s{mobdevtype}","module":"%s{module}","month":"%s{mon}","month_of_year":"%02d{mth}","nssserviceip":"%s{nsssvcip}","oapprulelabel":"%s{oapprulelabel}","obwclassname":"%s{obwclassname}","ocip":"%d{ocip}","ocpubip":"%d{ocpubip}","odevicehostname":"%s{odevicehostname}","odevicename":"%s{odevicename}","odeviceowner":"%s{odeviceowner}","odlpdict":"%s{odlpdict}","odlpeng":"%s{odlpeng}","odlprulename":"%s{odlprulename}","ofwd_gw_name":"%s{ofwd_gw_name}","ologin":"%s{ologin}","ordr_rulename":"%s{ordr_rulename}","ourlcat":"%s{ourlcat}","ourlfilterrulelabel":"%s{ourlfilterrulelabel}","ozpa_app_seg_name":"%s{ozpa_app_seg_name}","externalsslpolicyreason":"%s{externalspr}","productversion":"%s{productversion}","rdr_rulename":"%s{rdr_rulename}","refererhost":"%s{refererhost}","reqheadersize":"%d{reqhdrsize}","reqdatasize":"%d{reqdatasize}","respheadersize":"%d{resphdrsize}","respdatasize":"%d{respdatasize}","riskscore":"%d{riskscore}","ruletype":"%s{ruletype}","second":"%02d{ss}","srvcertchainvalpass":"%s{srvcertchainvalpass}","srvcertvalidationtype":"%s{srvcertvalidationtype}","srvcertvalidityperiod":"%s{srvcertvalidityperiod}","srvsslcipher":"%s{srvsslcipher}","serversslsessreuse":"%s{serversslsessreuse}","srvocspresult":"%s{srvocspresult}","srvtlsversion":"%s{srvtlsversion}","srvwildcardcert":"%s{srvwildcardcert}","ssldecrypted":"%s{ssldecrypted}","throttlereqsize":"%d{throttlereqsize}","throttlerespsize":"%d{throttlerespsize}","totalsize":"%d{totalsize}","trafficredirectmethod":"%s{trafficredirectmethod}","unscannabletype":"%s{unscannabletype}","upload_doctypename":"%s{upload_doctypename}","upload_fileclass":"%s{upload_fileclass}","upload_filetype":"%s{upload_filetype}","urlcatmethod":"%s{urlcatmethod}","urlsubcat":"%s{urlcat}","urlsupercat":"%s{urlsupercat}","urlclass":"%s{urlclass}","useragentclass":"%s{uaclass}","useragenttoken":"%s{ua_token}","userlocationname":"%s{userlocationname}","year":"%04d{yyyy}","ztunnelversion":"%s{ztunnelversion}","zpa_app_seg_name":"%s{zpa_app_seg_name}"\}\} ``` diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml index 5cd89f1134f..dde47f3d7b7 100644 --- a/packages/zscaler_zia/manifest.yml +++ b/packages/zscaler_zia/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.3" name: zscaler_zia title: Zscaler Internet Access -version: "3.0.3" +version: "3.0.4" description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent. type: integration categories: From ae4f22f6386d9f948f87e5ca5317329d41a5ed59 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 8 Aug 2024 17:09:03 +0200 Subject: [PATCH 12/13] updatecli: pass the username to fix the issue with auth (#10739) Pin version v2.64.0 Set username to solve the issues with the authentication needed. Simplify GitHub action to use a folder with all the pipelines Restructure the folder layout for the updatecli, updatecli.d for pipelines and values.d for configuration files. Co-authored-by: Victor Martinez --- .github/workflows/bump-elastic-stack-version.yml | 14 ++++---------- .../updatecli.d/bump-latest-7x-version.yml | 1 + .../updatecli.d/bump-latest-snapshot-version.yml | 1 + .../{updatecli.d => updatecli/values.d}/scm.yml | 0 4 files changed, 6 insertions(+), 10 deletions(-) rename .github/workflows/{ => updatecli}/updatecli.d/bump-latest-7x-version.yml (95%) rename .github/workflows/{ => updatecli}/updatecli.d/bump-latest-snapshot-version.yml (95%) rename .github/workflows/{updatecli.d => updatecli/values.d}/scm.yml (100%) diff --git a/.github/workflows/bump-elastic-stack-version.yml b/.github/workflows/bump-elastic-stack-version.yml index c3182297576..dc2957d8137 100644 --- a/.github/workflows/bump-elastic-stack-version.yml +++ b/.github/workflows/bump-elastic-stack-version.yml @@ -7,7 +7,7 @@ on: - cron: '0 1 * * 1-5' pull_request: paths: - - .github/updatecli.d/* + - .github/workflows/updatecli/** - .github/workflows/bump-elastic-stack-version.yml permissions: @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Updatecli in the runner - uses: updatecli/updatecli-action@v2.62.0 + uses: updatecli/updatecli-action@3a8785d88ec4fa03d86521a181f37c0e74627463 #v2.64.0 - name: Select diff action if: ${{ github.event_name == 'pull_request' }} @@ -37,14 +37,8 @@ jobs: run: | echo "UPDATECLI_ACTION=apply" >> $GITHUB_ENV - - name: Update latest testing 7.x stack version + - name: Update latest testing stack versions # --experimental needed for commitusingapi option. - run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli.d/bump-latest-7x-version.yml --values .github/workflows/updatecli.d/scm.yml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Update latest testing stack version - # --experimental needed for commitusingapi option. - run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli.d/bump-latest-snapshot-version.yml --values .github/workflows/updatecli.d/scm.yml + run: updatecli --experimental ${{ env.UPDATECLI_ACTION }} --config .github/workflows/updatecli/updatecli.d --values .github/workflows/updatecli/values.d/scm.yml env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/updatecli.d/bump-latest-7x-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml similarity index 95% rename from .github/workflows/updatecli.d/bump-latest-7x-version.yml rename to .github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml index 697b7d75b98..a5d83b4fa81 100644 --- a/.github/workflows/updatecli.d/bump-latest-7x-version.yml +++ b/.github/workflows/updatecli/updatecli.d/bump-latest-7x-version.yml @@ -19,6 +19,7 @@ scms: owner: '{{ .scm.owner }}' repository: '{{ .scm.repository }}' user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' token: '{{ requiredEnv "GITHUB_TOKEN" }}' commitusingapi: true branch: main diff --git a/.github/workflows/updatecli.d/bump-latest-snapshot-version.yml b/.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml similarity index 95% rename from .github/workflows/updatecli.d/bump-latest-snapshot-version.yml rename to .github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml index 65267d2403c..d4ca8b55c39 100644 --- a/.github/workflows/updatecli.d/bump-latest-snapshot-version.yml +++ b/.github/workflows/updatecli/updatecli.d/bump-latest-snapshot-version.yml @@ -19,6 +19,7 @@ scms: owner: '{{ .scm.owner }}' repository: '{{ .scm.repository }}' user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' token: '{{ requiredEnv "GITHUB_TOKEN" }}' commitusingapi: true branch: main diff --git a/.github/workflows/updatecli.d/scm.yml b/.github/workflows/updatecli/values.d/scm.yml similarity index 100% rename from .github/workflows/updatecli.d/scm.yml rename to .github/workflows/updatecli/values.d/scm.yml From 2db88f3f03b2488aecfac05af873eaa47923040c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 8 Aug 2024 17:26:02 +0200 Subject: [PATCH 13/13] [CI] Report skipped tests in junit annotations (#10735) Update jUnit buildkite plugin up to version 2.5.0, so skipped tests can be shown as part of the annotation created in each buildkite build. --- .buildkite/pipeline.serverless.yml | 4 +++- .buildkite/pipeline.yml | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml index 7c3d5c7bb91..f824c894ff4 100644 --- a/.buildkite/pipeline.serverless.yml +++ b/.buildkite/pipeline.serverless.yml @@ -76,8 +76,10 @@ steps: - label: ":junit: Junit annotate" plugins: - - junit-annotate#v2.4.1: + - junit-annotate#v2.5.0: artifacts: "build/test-results/*.xml" + failed-download-exit-code: 0 # Not fail the build in case there are no XML files + report-skipped: true agents: provider: "gcp" # junit plugin requires docker diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b43a25198ff..3f09cfe326b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -80,9 +80,10 @@ steps: - label: ":junit: Junit annotate" plugins: - - junit-annotate#v2.4.1: + - junit-annotate#v2.5.0: artifacts: "build/test-results/*.xml" failed-download-exit-code: 0 # Not fail the build in case there are no XML files + report-skipped: true agents: provider: "gcp" # junit plugin requires docker