diff --git a/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml b/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml index d58d935d01f..e6363558472 100644 --- a/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml +++ b/packages/checkpoint_harmony_endpoint/_dev/deploy/docker/files/config.yml @@ -1,97 +1,4 @@ rules: -#### Forensics #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxforensics"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxforensics", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxforensics" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Forensics\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxforensics" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxforensics - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxforensics" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "state": "Ready", - "pageTokens": [ - "xxxxforensics" - ] - } - } - - path: /app/laas-logs-api/api/logs_query/retrieve - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxforensics" - request_body: /^\{\"pageToken\":\"xxxxforensics\",\"taskId\":\"xxxxforensics\"\}/ - responses: - - status_code: 200 - body: | - { - "success": true, - "data": { - "records": [ - {"confidence_level": "High", "policy_date": "2024-09-02T06:23:25.0000000Z", "severity": "Critical", "time": "2024-09-03T08:53:12.000001Z", "id": "a4640108-91b1-0f19-66d6-ceb500000000", "orig": "164.100.1.8", "sequencenum": 1, "action": "Detect", "product": "Forensics", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "description": "To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c Attack status: Dormant.", "protection_type": "File Reputation", "attack_status": "Dormant", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "detected_by": "Endpoint File Reputation", "event_type": "Forensics Case Analysis", "file_md5": "1468c1908845ef238f7f196809946288", "file_name": [ "malz5.zip" ], "file_sha1": "62f0bd56b0e1235b99940b34916c19ecfac8e80c", "file_size": 12707198, "file_type": "zip", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "malware_action": [ " " ], "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "packet_capture_unique_id": "0acd55a9-f241-4097-a699-6b7e41cd26af", "policy_name": "Default Forensics settings", "policy_number": 3, "protection_name": "Gen.Rep.zip", "remediated_files": "malz5.zip(Remediation disabled in policy)", "resource": [ "c:\\users\\admin\\downloads\\malz5.zip" ], "service_domain": "ep-demo", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "suspicious_events": "System Shutdown / Reboot: ; ", "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "packet_capture": "Packet Capture"} - ], - "recordsCount": 1, - "nextPageToken": "NULL" - } - } #### Anti-Bot #### - path: /auth/external methods: ["POST"] @@ -100,7 +7,7 @@ rules: - "application/json" Content-Type: - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxantibot"}' + request_body: '{"accessKey":"testaccesskey","clientId":"testclientid"}' responses: - status_code: 200 headers: @@ -110,10 +17,10 @@ rules: { "success": true, "data": { - "token": "xxxxantibot", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 + "token": "testauthtoken", + "csrf": "testcsrftoken", + "expires": "Thu, 10 Sep 2099 09:11:32 GMT", + "expiresIn": 954000 } } - path: /app/laas-logs-api/api/logs_query @@ -124,7 +31,7 @@ rules: Content-Type: - "application/json" Authorization: - - "Bearer xxxxantibot" + - "Bearer testauthtoken" request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Anti-Bot\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ responses: - status_code: 200 @@ -135,202 +42,10 @@ rules: { "success": true, "data": { - "taskId": "xxxxantibot" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxantibot - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxantibot" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "state": "Ready", - "pageTokens": [ - "xxxxantibot" - ] - } - } - - path: /app/laas-logs-api/api/logs_query/retrieve - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxantibot" - request_body: /^\{\"pageToken\":\"xxxxantibot\",\"taskId\":\"xxxxantibot\"\}/ - responses: - - status_code: 200 - body: | - { - "success": true, - "data": { - "records": [ - {"confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "id": "a4640108-91b1-0f19-66da-bf6400000001", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" }, - {"confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:26Z", "id": "a4640108-91b1-0f19-66da-bf6400000000", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "bb7c48cddde076e7eb44022520f40f77", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "087e116c-c098-44ba-95dd-064a90189b4a", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "admin", "proxy_src_ip": "0.0.0.0" }, - {"confidence_level": "Medium", "policy_date": "2024-08-29T13:12:51.0000000Z", "severity": "Critical", "time": "2024-09-02T08:53:44Z", "id": "a4640108-91b1-0f19-66d5-7d9d00000000", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected bot activity [Anti-Bot test.TC.e]. To exclude: On the Harmony Endpoint Management add an exclusion of type \"URL\" with value: \"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 2, "protection_type": "URL Reputation", "file_md5": "bd075be9d011daaa82c3f9ff2572076e", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Communication with C&C" ], "packet_capture_unique_id": "6c239c74-89a9-4797-ab6b-75a2b2a6afd7", "protection_name": "Anti-Bot test.TC.e", "resource": [ "www.threat-cloud.com" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"URL\",\"exclusion_value\":{\"default_value\":\"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "89.160.20.128", "process_username": "admin", "proxy_src_ip": "89.160.20.128", "dst_country": [ "UnitedStates" ]} - ], - "recordsCount": 3, - "nextPageToken": "NULL" - } - } -#### Anti-Malware #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxantimalware"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxantimalware", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxantimalware" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Anti-Malware\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxantimalware" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxantimalware - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxantimalware" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "state": "Ready", - "pageTokens": [ - "xxxxantimalware" - ] - } - } - - path: /app/laas-logs-api/api/logs_query/retrieve - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxantimalware" - request_body: /^\{\"pageToken\":\"xxxxantimalware\",\"taskId\":\"xxxxantimalware\"\}/ - responses: - - status_code: 200 - body: | - { - "success": true, - "data": { - "records": [ - {"confidence_level": "High", "policy_date": "2024-08-29T13:12:46.0000000Z", "severity": "High", "time": "2024-09-02T09:09:07Z", "id": "a4640108-91b1-0f19-66d5-815d0000000f", "orig": "164.100.1.8", "sequencenum": 16777215, "product": "Anti-Malware", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "event_type": "Infection", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Anti-Malware settings for the entire organization", "policy_number": 3, "action_details": "Infected", "connectivity_state": "Connected", "engine_ver": "3.90", "sig_ver": "202409011444", "action": "Detect", "protection_type": "Protection", "file_name": [ "9e68140d-22bb-4e96-8aaa-70ec80eb2dc4.tmp" ], "packet_capture_unique_id": "31dc576b-7192-49bf-b2fc-b40c93f84b7c", "protection_name": "Mal/ShellDl-A", "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"File & Folder exclusions (system, scheduled and on-demand)\",\"exclusion_type\":\"Path\",\"exclusion_value\":{\"default_value\":\"md5:\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"md5 taken from file C:\\\\Users\\\\admin\\\\AppData\\\\Local\\\\Temp\\\\9e68140d-22bb-4e96-8aaa-70ec80eb2dc4.tmp\"}}]", "infection_category": "Malware" }, - {"confidence_level": "High", "policy_date": "2024-09-06T10:21:35.0000000Z", "severity": "High", "time": "2024-09-09T12:38:42Z", "id": "a4640108-91b1-0f19-66de-ed0700000012", "orig": "164.100.1.8", "sequencenum": 16777215, "product": "Anti-Malware", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "event_type": "Infection", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Anti-Malware settings for the entire organization", "policy_number": 7, "action_details": "Infected", "connectivity_state": "Connected", "engine_ver": "3.90", "sig_ver": "202409090014", "action": "Detect", "protection_type": "Protection", "file_md5": "44d88612fea8a8f36de82e1278abb02f", "file_name": [ "eicar.txt" ], "packet_capture_unique_id": "bb6c808c-b96e-4f12-9749-1dfae7f6f8c2", "protection_name": "EICAR-AV-Test", "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"File & Folder exclusions (system, scheduled and on-demand)\",\"exclusion_type\":\"Path\",\"exclusion_value\":{\"default_value\":\"md5:44d88612fea8a8f36de82e1278abb02f\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"md5 taken from file C:\\\\Users\\\\admin\\\\Downloads\\\\eicar.txt\"}}]", "infection_category": "Virus" }, - {"policy_date": "2024-08-29T13:12:46.0000000Z", "severity": "Low", "time": "2024-09-02T10:37:24Z", "id": "a4640108-91b1-0f19-66d5-967600000000", "orig": "164.100.1.8", "sequencenum": 16777215, "product": "Anti-Malware", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "event_type": "Scan Stop", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Anti-Malware settings for the entire organization", "policy_number": 3, "action_details": "Finished", "connectivity_state": "Connected", "engine_ver": "3.90", "sig_ver": "202409011444", "integrity_av_invoke_type": "Scheduled", "duration": 2787, "items_detected": 0, "items_scanned": 61330, "items_treated": 0 }, - {"policy_date": "2024-09-03T10:46:10.0000000Z", "severity": "Low", "time": "2024-09-03T11:27:05.000002Z", "id": "a4640108-91b1-0f19-66d6-f30b00000004", "orig": "164.100.1.8", "sequencenum": 2, "product": "Anti-Malware", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "event_type": "Scan Start", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Anti-Malware settings for the entire organization", "policy_number": 4, "action_details": "Started", "connectivity_state": "Connected", "engine_ver": "3.90", "sig_ver": "202409021821", "integrity_av_invoke_type": "Unknown" }, - {"policy_date": "2024-09-06T10:21:35.0000000Z", "severity": "Low", "time": "2024-09-09T12:38:48Z", "id": "a4640108-91b1-0f19-66de-ed0700000013", "orig": "164.100.1.8", "sequencenum": 16777215, "product": "Anti-Malware", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "event_type": "Update", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Anti-Malware settings for the entire organization", "policy_number": 7, "action_details": "Successfully", "connectivity_state": "Connected", "engine_ver": "3.90", "failed_updates": 0, "result": "Finished", "sig_ver": "202409090014"} - ], - "recordsCount": 5, - "nextPageToken": "NULL" - } - } -#### Threat Emulation #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxthreatemulation"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxthreatemulation", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxthreatemulation" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Threat Emulation\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxthreatemulation" + "taskId": "testtaskid1" } } - - path: /app/laas-logs-api/api/logs_query/xxxxthreatemulation + - path: /app/laas-logs-api/api/logs_query/testtaskid1 methods: ["GET"] request_headers: Accept: @@ -338,7 +53,7 @@ rules: Content-Type: - "application/json" Authorization: - - "Bearer xxxxthreatemulation" + - "Bearer testauthtoken" responses: - status_code: 200 headers: @@ -348,93 +63,10 @@ rules: { "success": true, "data": { - "state": "Ready", - "pageTokens": [ - "xxxxthreatemulation" - ] + "state": "Processing", + "pageTokens": [] } } - - path: /app/laas-logs-api/api/logs_query/retrieve - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxthreatemulation" - request_body: /^\{\"pageToken\":\"xxxxthreatemulation\",\"taskId\":\"xxxxthreatemulation\"\}/ - responses: - - status_code: 200 - body: | - { - "success": true, - "data": { - "records": [ - {"confidence_level": "High", "policy_date": "2024-09-03T10:46:13.0000000Z", "severity": "Low", "time": "2024-09-06T09:06:55Z", "id": "a4640108-91b1-0f19-66da-c6a900000004", "orig": "164.100.1.8", "sequencenum": 16777215, "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Download of malicious file eicar_com.zip was prevented . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: d2726507-4c9eac2e-2122ed69-294dbc4d-7cce9141", "event_type": "TE Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", "policy_number": 4, "product": "Threat Emulation", "product_family": "Endpoint", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "domain": "SMC User", "type": "Log", "action": "Prevent", "protection_type": "HTTP Emulation", "file_name": [ "eicar_com.zip" ], "file_sha1": "d27265074c9eac2e2122ed69294dbc4d7cce9141", "file_size": 184, "file_type": "zip", "malware_action": [ " " ], "packet_capture_unique_id": "9672a696-fdcd-43be-a8c0-7b50b5efc6c4", "protection_name": "gen.ba.sb.zip", "resource": [ "https://secure.eicar.org/eicar_com.zip" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"secure.eicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "analyzed_on": "Harmony Local Cache", "incident_uid": "c40d72fb-533b-423a-9749-d239a9c4ea3f", "verdict": "Malicious", "web_client_type": [ "Chrome" ] }, - {"confidence_level": "High", "policy_date": "2024-08-29T13:12:50.0000000Z", "severity": "High", "time": "2024-09-03T08:52:45Z", "id": "a4640108-91b1-0f19-66d6-cf6900000000", "orig": "164.100.1.8", "sequencenum": 16777215, "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Endpoint FR detected malicious file (malz5.zip) . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 62f0bd56-b0e1235b-99940b34-916c19ec-fac8e80c", "event_type": "TE Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", "policy_number": 3, "product": "Threat Emulation", "product_family": "Endpoint", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "domain": "SMC User", "type": "Log", "action": "Detect", "protection_type": "File Reputation", "file_md5": "1468c1908845ef238f7f196809946288", "file_name": [ "malz5.zip" ], "file_sha1": "62f0bd56b0e1235b99940b34916c19ecfac8e80c", "file_size": 12707198, "file_type": "zip", "malware_action": [ " " ], "packet_capture_unique_id": "ee12249f-b2bd-48aa-80a4-9f92745caba1", "protection_name": "Gen.Rep.zip", "resource": [ "C:\\Users\\admin\\Downloads\\malz5.zip" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"SHA1\",\"exclusion_value\":{\"default_value\":\"62f0bd56b0e1235b99940b34916c19ecfac8e80c\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "analyzed_on": "Harmony Local Cache", "incident_uid": "0acd55a9-f241-4097-a699-6b7e41cd26af", "verdict": "Malicious", "web_client_type": [ " " ] }, - {"confidence_level": "High", "policy_date": "2024-08-29T13:12:50.0000000Z", "severity": "Critical", "time": "2024-09-02T09:04:54Z", "id": "a4640108-91b1-0f19-66d5-803100000012", "orig": "164.100.1.8", "sequencenum": 16777215, "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Endpoint TE detected malicious file (681573a2-414a-4f7d-9683-177df4f8ca7f.tmp) . To exclude the file: On the Harmony Endpoint Management add this sha1 exclusion: 9d3395d9-4c6bbba5-2abf0e6a-fcbf4ca3-12597c21", "event_type": "TE Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", "policy_number": 3, "product": "Threat Emulation", "product_family": "Endpoint", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "domain": "SMC User", "type": "Log", "action": "Detect", "protection_type": "File System Emulation", "file_md5": "ebe8b633d231bbfee9543d744a2ab59d", "file_name": [ "681573a2-414a-4f7d-9683-177df4f8ca7f.tmp" ], "file_sha1": "9d3395d94c6bbba52abf0e6afcbf4ca312597c21", "file_size": 139648, "file_type": "zip", "malware_action": [ "Adware\",\"Solimba\",\"Trojan\",\"behavior" ], "packet_capture_unique_id": "5e3302e5-3f73-4b77-beec-2849003e9d47", "protection_name": "Gen.SB.zip", "resource": [ "C:\\Users\\admin\\Downloads\\681573a2-414a-4f7d-9683-177df4f8ca7f.tmp" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"SHA1\",\"exclusion_value\":{\"default_value\":\"9d3395d94c6bbba52abf0e6afcbf4ca312597c21\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "analyzed_on": "Check Point Threat Emulation Cloud", "incident_uid": "74a33ecb-1b91-4c25-a136-1989eb175638", "verdict": "Malicious", "web_client_type": [ " " ]} - ], - "recordsCount": 3, - "nextPageToken": "NULL" - } - } -#### Threat Extraction #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxthreatextraction"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxthreatextraction", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxthreatextraction" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Threat Extraction\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxthreatextraction" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxthreatextraction - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxthreatextraction" - responses: - status_code: 200 headers: Content-Type: @@ -445,7 +77,7 @@ rules: "data": { "state": "Ready", "pageTokens": [ - "xxxxthreatextraction" + "testpagetoken1" ] } } @@ -457,8 +89,8 @@ rules: Content-Type: - "application/json" Authorization: - - "Bearer xxxxthreatextraction" - request_body: /^\{\"pageToken\":\"xxxxthreatextraction\",\"taskId\":\"xxxxthreatextraction\"\}/ + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken1\",\"taskId\":\"testtaskid1\"\}/ responses: - status_code: 200 body: | @@ -466,174 +98,19 @@ rules: "success": true, "data": { "records": [ - {"confidence_level": "High", "policy_date": "2024-08-29T13:12:50.0000000Z", "severity": "Informational", "time": "2024-09-02T09:21:42.000001Z", "id": "a4640108-91b1-0f19-66d5-83f100000019", "orig": "164.100.1.8", "sequencenum": 1, "action": "Extract", "product": "Threat Extraction", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "File is not supported for extraction", "event_type": "TEX Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", "policy_number": 3, "protection_type": "Content Removal", "advanced_info": " \"disable_exclusion\": true ", "extension_version": " 990.97.911 ", "file_name": [ "mirai.sh4" ], "file_sha1": "no-sha1", "file_size": 0, "file_type": "sh4", "malware_action": [ "Not Supported" ], "protection_name": "Extract potentially malicious content", "resource": [ "blob:https://github.com/6bd30ea7-29a8-4dd2-9056-f5077632e110" ], "web_client_type": [ "Chrome" ]} + {"id": "a4640108-91b1-0f19-66da-bf6400000001", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66da-bf6400000002", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:26Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "bb7c48cddde076e7eb44022520f40f77", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "087e116c-c098-44ba-95dd-064a90189b4a", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "admin", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66d5-7d9d00000003", "confidence_level": "Medium", "policy_date": "2024-08-29T13:12:51.0000000Z", "severity": "Critical", "time": "2024-09-02T08:53:44Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected bot activity [Anti-Bot test.TC.e]. To exclude: On the Harmony Endpoint Management add an exclusion of type \"URL\" with value: \"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 2, "protection_type": "URL Reputation", "file_md5": "bd075be9d011daaa82c3f9ff2572076e", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Communication with C&C" ], "packet_capture_unique_id": "6c239c74-89a9-4797-ab6b-75a2b2a6afd7", "protection_name": "Anti-Bot test.TC.e", "resource": [ "www.threat-cloud.com" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"URL\",\"exclusion_value\":{\"default_value\":\"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "89.160.20.128", "process_username": "admin", "proxy_src_ip": "89.160.20.128", "dst_country": [ "UnitedStates" ]}, + {"id": "a4640108-91b1-0f19-66da-bf6400000004", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66da-bf6400000005", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:26Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "bb7c48cddde076e7eb44022520f40f77", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "087e116c-c098-44ba-95dd-064a90189b4a", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "admin", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66d5-7d9d00000006", "confidence_level": "Medium", "policy_date": "2024-08-29T13:12:51.0000000Z", "severity": "Critical", "time": "2024-09-02T08:53:44Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected bot activity [Anti-Bot test.TC.e]. To exclude: On the Harmony Endpoint Management add an exclusion of type \"URL\" with value: \"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 2, "protection_type": "URL Reputation", "file_md5": "bd075be9d011daaa82c3f9ff2572076e", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Communication with C&C" ], "packet_capture_unique_id": "6c239c74-89a9-4797-ab6b-75a2b2a6afd7", "protection_name": "Anti-Bot test.TC.e", "resource": [ "www.threat-cloud.com" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"URL\",\"exclusion_value\":{\"default_value\":\"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "89.160.20.128", "process_username": "admin", "proxy_src_ip": "89.160.20.128", "dst_country": [ "UnitedStates" ]}, + {"id": "a4640108-91b1-0f19-66da-bf6400000007", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66da-bf6400000008", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:26Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "bb7c48cddde076e7eb44022520f40f77", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "087e116c-c098-44ba-95dd-064a90189b4a", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "admin", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66d5-7d9d00000009", "confidence_level": "Medium", "policy_date": "2024-08-29T13:12:51.0000000Z", "severity": "Critical", "time": "2024-09-02T08:53:44Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected bot activity [Anti-Bot test.TC.e]. To exclude: On the Harmony Endpoint Management add an exclusion of type \"URL\" with value: \"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 2, "protection_type": "URL Reputation", "file_md5": "bd075be9d011daaa82c3f9ff2572076e", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Communication with C&C" ], "packet_capture_unique_id": "6c239c74-89a9-4797-ab6b-75a2b2a6afd7", "protection_name": "Anti-Bot test.TC.e", "resource": [ "www.threat-cloud.com" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"URL\",\"exclusion_value\":{\"default_value\":\"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "89.160.20.128", "process_username": "admin", "proxy_src_ip": "89.160.20.128", "dst_country": [ "UnitedStates" ]}, + {"id": "a4640108-91b1-0f19-66da-bf6400000010", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" } ], - "recordsCount": 1, - "nextPageToken": "NULL" - } - } -#### URL Filtering #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxurlfiltering"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxurlfiltering", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxurlfiltering" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"URL Filtering\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxurlfiltering" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxurlfiltering - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxurlfiltering" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "state": "Ready", - "pageTokens": [ - "xxxxurlfiltering" - ] - } - } - - path: /app/laas-logs-api/api/logs_query/retrieve - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxurlfiltering" - request_body: /^\{\"pageToken\":\"xxxxurlfiltering\",\"taskId\":\"xxxxurlfiltering\"\}/ - responses: - - status_code: 200 - body: | - { - "success": true, - "data": { - "records": [ - {"policy_date": "2024-09-06T10:08:44.0000000Z", "severity": "Informational", "time": "2024-09-06T10:19:56.000001Z", "id": "a4640108-91b1-0f19-66da-d78900000009", "orig": "164.100.1.8", "sequencenum": 1, "action": "Prevent", "product": "URL Filtering", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "To exclude: Open the Harmony Management -> POLICY -> Threat Prevention -> EXCLUSION CENTER -> Web and Files Protection -> URL Filtering exclusions -> + -> paste this: livingliquidz.com", "event_type": "URLF Info Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 5, "protection_type": "URL Filtering", "protection_name": "gen.urlf", "resource": [ "https://livingliquidz.com/" ], "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"URL Filtering exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"livingliquidz.com\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "web_client_type": [ "Chrome" ], "app_id": "0", "app_properties": [ "Alcohol & Tobacco" ], "appi_name": "livingliquidz.com", "matched_category": "Alcohol & Tobacco", "process_exe_path": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", "usercheck_incident_uid": "7fcc5b44" }, - {"policy_date": "2024-09-06T09:57:28.0000000Z", "severity": "Informational", "time": "2024-09-06T10:07:43Z", "id": "a4640108-91b1-0f19-66da-d62100000013", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "URL Filtering", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "URLF Info Event", "event_type": "URLF Info Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 4, "protection_type": "URL Filtering", "protection_name": "gen.urlf", "resource": [ "https://secure.indeed.com/auth?branding=save-profile-modal&tmpl=inline&from=act_zeroauth_profile_tst&iframe_tk=9a019527-a6f1-4b3d-b803-2b25bb46b1db&hl=en_IN&co=IN" ], "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"URL Filtering exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"secure.indeed.com\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "web_client_type": [ "Chrome" ], "app_id": "0", "app_properties": [ "Job Search / Careers, Business / Economy" ], "appi_name": "secure.indeed.com", "matched_category": "Job Search / Careers", "process_exe_path": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", "usercheck_incident_uid": "b04d8940"} - ], - "recordsCount": 2, - "nextPageToken": "NULL" - } - } -#### Zero Phishing #### - - path: /auth/external - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - request_body: '{"accessKey":"xxxx","clientId":"xxxxzerophishing"}' - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "token": "xxxxzerophishing", - "csrf": "xxxx", - "expires": "Tue, 10 Sep 2024 09:11:32 GMT", - "expiresIn": 1800 - } - } - - path: /app/laas-logs-api/api/logs_query - methods: ["POST"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxzerophishing" - request_body: /^\{\"cloudService\":\"Harmony Endpoint\",\"filter\":\"product:\\\"Zero Phishing\\\"\",\"limit\":[0-9]+,\"pageLimit\":[0-9]+,\"timeframe\":\{\"endTime\":\"[0-9T\:\-Z]+\",\"startTime\":\"[0-9T\:\-Z]+\"\}\}/ - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "taskId": "xxxxzerophishing" - } - } - - path: /app/laas-logs-api/api/logs_query/xxxxzerophishing - methods: ["GET"] - request_headers: - Accept: - - "application/json" - Content-Type: - - "application/json" - Authorization: - - "Bearer xxxxzerophishing" - responses: - - status_code: 200 - headers: - Content-Type: - - "application/json" - body: | - { - "success": true, - "data": { - "state": "Ready", - "pageTokens": [ - "xxxxzerophishing" - ] + "recordsCount": 10, + "nextPageToken": "testpagetoken2" } } - path: /app/laas-logs-api/api/logs_query/retrieve @@ -644,8 +121,8 @@ rules: Content-Type: - "application/json" Authorization: - - "Bearer xxxxzerophishing" - request_body: /^\{\"pageToken\":\"xxxxzerophishing\",\"taskId\":\"xxxxzerophishing\"\}/ + - "Bearer testauthtoken" + request_body: /^\{\"pageToken\":\"testpagetoken2\",\"taskId\":\"testtaskid1\"\}/ responses: - status_code: 200 body: | @@ -653,9 +130,12 @@ rules: "success": true, "data": { "records": [ - {"confidence_level": "High", "policy_date": "2024-08-29T13:12:50.0000000Z", "severity": "High", "time": "2024-09-02T08:51:08Z", "id": "a4640108-91b1-0f19-66d5-7d6100000004", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Zero Phishing", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Deceptive site (https://main.sbm-demo.xyz/zero-phishing) was detected.", "event_type": "Phishing Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_name": "Default Threat Extraction, Emulation and Anti-Exploit settings for the entire organization", "policy_number": 3, "protection_type": "Phishing", "malware_action": [ " " ], "protection_name": "gen.ba.phishing", "resource": [ "https://main.sbm-demo.xyz/zero-phishing" ], "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Threat Emulation, Extraction and Zero Phishing Exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"main.sbm-demo.xyz\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "extension_version": "Check Point Endpoint Security Client", "web_client_type": [ "Chrome" ]} + {"id": "a4640108-91b1-0f19-66da-bf6400000011", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66da-bf6400000012", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:26Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "bb7c48cddde076e7eb44022520f40f77", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "087e116c-c098-44ba-95dd-064a90189b4a", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "admin", "proxy_src_ip": "0.0.0.0" }, + {"id": "a4640108-91b1-0f19-66d5-7d9d00000013", "confidence_level": "Medium", "policy_date": "2024-08-29T13:12:51.0000000Z", "severity": "Critical", "time": "2024-09-02T08:53:44Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected bot activity [Anti-Bot test.TC.e]. To exclude: On the Harmony Endpoint Management add an exclusion of type \"URL\" with value: \"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 2, "protection_type": "URL Reputation", "file_md5": "bd075be9d011daaa82c3f9ff2572076e", "file_name": [ "chrome.exe" ], "file_size": 2742376, "file_type": "exe", "malware_action": [ "Communication with C&C" ], "packet_capture_unique_id": "6c239c74-89a9-4797-ab6b-75a2b2a6afd7", "protection_name": "Anti-Bot test.TC.e", "resource": [ "www.threat-cloud.com" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"URL\",\"exclusion_value\":{\"default_value\":\"http://www.threat-cloud.com/test/files/MediumConfidenceBot.html\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "89.160.20.128", "process_username": "admin", "proxy_src_ip": "89.160.20.128", "dst_country": [ "UnitedStates" ]}, + {"id": "a4640108-91b1-0f19-66da-bf6400000014", "confidence_level": "High", "policy_date": "2024-09-03T10:46:14.0000000Z", "severity": "High", "time": "2024-09-06T08:32:28Z", "orig": "164.100.1.8", "sequencenum": 16777215, "action": "Detect", "product": "Anti-Bot", "domain": "SMC User", "product_family": "Endpoint", "type": "Log", "client_name": "Check Point Endpoint Security Client", "client_version": [ "88.50.0213" ], "description": "Detected dns query to malicious domain malware.wicar.org [Infecting Website.RS.TC.32eclVjT]. To exclude: On the Harmony Endpoint Management add an Anti-Bot exclusion of type Domain with value: \"malware.wicar.org\"", "event_type": "Anti Bot Event", "host_type": [ "Desktop" ], "installed_products": "Full Disk Encryption; Media Encryption & Port Protection; Firewall; Compliance; Application Control; Anti-Malware; VPN; Anti-Bot; Forensics; Threat Emulation", "os_name": [ "Microsoft Windows 10 Pro" ], "os_version": [ "10.0-19045-SP0.0-SMP" ], "policy_name": "Default Anti-Bot settings", "src": "10.35.38.102", "src_machine_name": "DESKTOP-E2P4OL0", "src_user_name": [ "admin" ], "tenant_id": "3e15ed24-89ff-4986-a204-c425cee4ba48", "user_sid": "S-1-5-21-3766288932-3295778425-2939962592-1001", "policy_number": 3, "protection_type": "URL Reputation", "file_md5": "7469cc568ad6821fd9d925542730a7d8", "file_name": [ "svchost.exe" ], "file_size": 47040, "file_type": "exe", "malware_action": [ "Access to site known to contain malware" ], "packet_capture_unique_id": "e71ae8af-f40e-49cb-9bef-1a91602f8faf", "protection_name": "Infecting Website.RS.TC.32eclVjT", "resource": [ "malware.wicar.org" ], "packet_capture": "Packet Capture", "advanced_info": "\"exclusions\":[{\"exclusion_engine_type\":\"Anti Bot exclusions\",\"exclusion_type\":\"Domain\",\"exclusion_value\":{\"default_value\":\"malware.wicar.org\",\"md5\":\"\",\"original_name\":\"\",\"signer\":\"\",\"process\":\"\",\"protection\":\"\",\"comment\":\"\"}}]", "dst": "0.0.0.0", "process_username": "NETWORK SERVICE", "proxy_src_ip": "0.0.0.0" } ], - "recordsCount": 1, + "recordsCount": 4, "nextPageToken": "NULL" } } diff --git a/packages/checkpoint_harmony_endpoint/changelog.yml b/packages/checkpoint_harmony_endpoint/changelog.yml index 513f29a6a60..129af0b9749 100644 --- a/packages/checkpoint_harmony_endpoint/changelog.yml +++ b/packages/checkpoint_harmony_endpoint/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.1" + changes: + - description: Auth and pagination fixes. + type: bugfix + link: https://github.com/elastic/integrations/pull/12158 - version: "0.2.0" changes: - description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error". diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antibot/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/antibot/_dev/test/system/test-default-config.yml index 3c721d139ba..b3867f415a9 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antibot/_dev/test/system/test-default-config.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/antibot/_dev/test/system/test-default-config.yml @@ -2,11 +2,11 @@ input: cel service: harmony vars: base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxantibot - access_key: xxxx - initial_interval: 10s - interval: 10s + client_id: testclientid + access_key: testaccesskey + initial_interval: 720h + interval: 5m limit: 10000 - page_limit: 100 + page_limit: 10 assert: - hit_count: 3 + hit_count: 14 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antibot/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/antibot/manifest.yml index 88607da6520..d5d5f2a54cd 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antibot/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/antibot/manifest.yml @@ -21,6 +21,22 @@ streams: required: true description: Request filter. Do not modify. show_user: false + - name: resource_rate_limit_limit + type: text + title: Resource Rate Limit + default: "0.2" + description: In requests per second. This controls polling frequency. + show_user: false + multi: false + required: true + - name: resource_rate_limit_burst + type: integer + title: Resource Rate Limit Burst + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. + show_user: false + multi: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml deleted file mode 100644 index 15794cea1bd..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxantimalware - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 5 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/manifest.yml index 6d9dfca7b0e..c86917f0270 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/manifest.yml @@ -24,17 +24,19 @@ streams: - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml deleted file mode 100644 index 9be446f0c53..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/forensics/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxforensics - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 1 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/forensics/manifest.yml index 04bf439bfcd..ec7661322ac 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/forensics/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/forensics/manifest.yml @@ -24,17 +24,19 @@ streams: - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml deleted file mode 100644 index 45b25e7957e..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxthreatemulation - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 3 diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/manifest.yml index dfdda371edc..9b35507f066 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/manifest.yml @@ -20,21 +20,23 @@ streams: default: "product:\"Threat Emulation\"" required: true description: Request filter. Do not modify. - show_user: false + show_user: false - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml deleted file mode 100644 index e348be21af4..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxthreatextraction - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 1 \ No newline at end of file diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/manifest.yml index 5c720eb9ebf..44abc52a535 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/manifest.yml @@ -24,17 +24,19 @@ streams: - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml deleted file mode 100644 index dea5efcb8bd..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxurlfiltering - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 2 \ No newline at end of file diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/manifest.yml index 87a248d2c77..d1eaa0709c2 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/manifest.yml @@ -24,17 +24,19 @@ streams: - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml deleted file mode 100644 index 94e8b25033e..00000000000 --- a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/_dev/test/system/test-default-config.yml +++ /dev/null @@ -1,12 +0,0 @@ -input: cel -service: harmony -vars: - base_url: http://{{Hostname}}:{{Port}} - client_id: xxxxzerophishing - access_key: xxxx - initial_interval: 10s - interval: 10s - limit: 10000 - page_limit: 100 -assert: - hit_count: 1 \ No newline at end of file diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs index 105bb8622fa..414cdcb1e8e 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs +++ b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs @@ -1,6 +1,6 @@ config_version: 3 -resource.rate_limit.limit: 0.2 -resource.rate_limit.burst: 5 +resource.rate_limit.limit: {{resource_rate_limit_limit}} +resource.rate_limit.burst: {{resource_rate_limit_burst}} {{#if enable_request_tracer}} resource.tracer.filename: "../../logs/cel/http-request-trace-*.ndjson" resource.tracer.maxbackups: 5 @@ -15,9 +15,15 @@ state: page_limit: {{page_limit}} filter: {{filter}} program: |- - (!has(state.cursor) || has(state.cursor) && has(state.cursor.auth_token) && state.cursor.auth_token == null) ? - ( - // Authenticating using API to retrieve auth token + + ( + state.?cursor.auth_data.expires.optMap(t, + t.parse_time(time_layout.RFC1123) - now() > duration("5m") + ).orValue(false) ? + // Current auth data exists - Use it. + state.cursor.auth_data + : + // No current auth data - Use credentials to fetch a new token. request("POST", state.url.trim_right("/") + "/auth/external").with( { "Header": { @@ -30,22 +36,28 @@ program: |- }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - body.data.token - ) - : - bytes(resp.Body).decode_json().as(body, - body.message - ) - ).as(auth_token, - // submit logs query to search security event logs + bytes(resp.Body).decode_json().as(body, + { + "token": body.data.token, + "expires": body.data.expires, + } + ) + ) + ).as(auth_data, + (state.?cursor.task_id.orValue(null) == null) ? + // No task ID - Submit a query and get its task ID. + { + "startTime": state.?cursor.next_startTime.orValue( + timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) + ), + "endTime": timestamp(now() - duration("1m")).format(time_layout.RFC3339), + }.as(timeframe, request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query").with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + auth_token], + "Authorization": ["Bearer " + auth_data.token], }, "Body": { "filter": state.filter, @@ -53,164 +65,168 @@ program: |- "pageLimit": state.page_limit, "cloudService": "Harmony Endpoint", "timeframe": { - "startTime": (state.?cursor.next_startTime.orValue(null) == null) ? - timestamp(now() - duration(state.initial_interval)).format(time_layout.RFC3339) - : - timestamp(state.cursor.next_startTime).format(time_layout.RFC3339), - "endTime": timestamp(now().format(time_layout.RFC3339)), + "startTime": timeframe.startTime, + "endTime": timeframe.endTime, }, }.encode_json(), } ).do_request().as(resp, - (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": { - "auth_token": auth_token, - "task_id": body.data.taskId, - "task_ready": false, - "page_token": null, - "next_startTime": (has(state.cursor) && has(state.cursor.next_startTime)) ? state.cursor.next_startTime : null, - "last_page": false, - }, - } - ) - ) - : + (resp.StatusCode != 200) ? + // Any error - We're at the start, so clear everything and retry after interval. state.with( { - "events": { - "error": { - "message": "Error " + bytes(resp.Body).decode_json().as(body, body.message), - }, - }, + "events": {"error": {"message": "Error response: " + string(resp.Body)}}, "want_more": false, "cursor": state.cursor.with( { - "auth_token": null, + "auth_data": null, "task_id": null, - "task_ready": false, "page_token": null, - "last_page": false, } ), } ) + : + // Query submitted - Save the task ID. + bytes(resp.Body).decode_json().as(body, + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": { + "auth_data": auth_data, + "task_id": body.data.taskId, + "page_token": null, + "next_startTime": timeframe.endTime, + }, + } + ) + ) ) ) - ) - : (has(state.cursor) && has(state.cursor.task_ready) && state.cursor.task_ready == false) ? - ( - // submit task ID to Check the progress of specific search event logs task and get the pageTokens + : (state.?cursor.page_token.orValue(null) == null) ? + // Task exists with no page token - Check whether it's ready or done. request("GET", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/" + state.cursor.task_id).with( { "Header": { "Accept": ["application/json"], "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], + "Authorization": ["Bearer " + auth_data.token], }, } ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - // 'Ready' - Found results. Ready to retrieve event logs records on the 1st page - (body.data.state == "Ready") ? - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": true, - "page_token": body.data.pageTokens[0], - "last_page": null, - } - ), - } - ) - : (body.data.state == "Done") ? - // 'Done' - The entire specified time range of request has been covered / No results have been found for specified request - state.with( - { - "events": [], - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), - } - ) - : - state.with( - { - "events": [{ "message": { "event": { "reason": "polling" }}.encode_json() }], - "want_more": true, - "cursor": state.cursor.with( - { - "task_ready": false, - "page_token": null, - "last_page": null, - } - ), - } - ) - ) - ) - ) - : - // use task ID and pageToken to retrieve event logs results on a specific page - request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( - { - "Header": { - "Accept": ["application/json"], - "Content-Type": ["application/json"], - "Authorization": ["Bearer " + state.cursor.auth_token], - }, - "Body": { - "taskId": state.cursor.task_id, - "pageToken": state.cursor.page_token, - }.encode_json(), - } - ).do_request().as(resp, - bytes(resp.Body).decode_json().as(body, - (body.data.nextPageToken == "NULL") ? + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), - "want_more": false, - "cursor": state.cursor.with( - { - "auth_token": null, - "task_id": null, - "task_ready": null, - "page_token": null, - "last_page": null, - } - ), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with({"auth_data": null}), } ) : + bytes(resp.Body).decode_json().as(body, + (body.data.state == "Ready") ? + // 'Ready' (Results found) - Save the first page token. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.pageTokens[0], + } + ), + } + ) + : (body.data.state == "Done") ? + // 'Done' (Results empty) - Clear the task ID and end the sequence. + state.with( + { + "events": [], + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + } + ), + } + ) + : + // Not ready or done - Keep polling. + state.with( + { + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + } + ), + } + ) + ) + ) + : + // Task is ready - Use the task ID and page token to retrieve a page of results. + request("POST", state.url.trim_right("/") + "/app/laas-logs-api/api/logs_query/retrieve").with( + { + "Header": { + "Accept": ["application/json"], + "Content-Type": ["application/json"], + "Authorization": ["Bearer " + auth_data.token], + }, + "Body": { + "taskId": state.cursor.task_id, + "pageToken": state.cursor.page_token, + }.encode_json(), + } + ).do_request().as(resp, + (resp.StatusCode == 401) ? + // 401 Unauthorized - Clear the auth data and retry immediately. state.with( { - "events": body.data.records.map(e, {"message": e.encode_json()}), + "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}], "want_more": true, - "cursor": state.cursor.with( + "cursor": state.cursor.with({"auth_data": null}), + } + ) + : + bytes(resp.Body).decode_json().as(body, + (body.data.nextPageToken != "NULL") ? + // Not the last page - Save the next page token and continue. + state.with( { - "task_ready": true, - "page_token": body.data.nextPageToken, - "last_page": false, + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": true, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "page_token": body.data.nextPageToken, + } + ), } - ), - } + ) + : + // Last page - Clear the task ID and page token, and end the sequence. + state.with( + { + "events": body.data.records.map(e, {"message": e.encode_json()}), + "want_more": false, + "cursor": state.cursor.with( + { + "auth_data": auth_data, + "task_id": null, + "page_token": null, + } + ), + } + ) ) ) - ) + ) tags: {{#if preserve_original_event}} - preserve_original_event @@ -224,4 +240,4 @@ publisher_pipeline.disable_host: true {{#if processors}} processors: {{processors}} -{{/if}} \ No newline at end of file +{{/if}} diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/manifest.yml b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/manifest.yml index 9ccf39ba34f..c653ff73434 100644 --- a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/manifest.yml @@ -24,17 +24,19 @@ streams: - name: resource_rate_limit_limit type: text title: Resource Rate Limit - description: The value of the response that specifies the total limit. + default: "0.2" + description: In requests per second. This controls polling frequency. show_user: false multi: false - required: false + required: true - name: resource_rate_limit_burst - type: text + type: integer title: Resource Rate Limit Burst - description: The maximum burst size. Burst is the maximum number of resource requests that can be made above the overall rate limit. + default: 5 + description: In requests. The maximum number of requests that can be immediately made following an idle period. show_user: false multi: false - required: false + required: true - name: preserve_original_event type: bool title: Preserve original event diff --git a/packages/checkpoint_harmony_endpoint/manifest.yml b/packages/checkpoint_harmony_endpoint/manifest.yml index e613acf62d5..2326fa3ce06 100644 --- a/packages/checkpoint_harmony_endpoint/manifest.yml +++ b/packages/checkpoint_harmony_endpoint/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.2.1 name: checkpoint_harmony_endpoint title: "Check Point Harmony Endpoint" -version: 0.2.0 +version: 0.2.1 source: license: "Elastic-2.0" description: "Collect logs from Check Point Harmony Endpoint" @@ -61,7 +61,7 @@ policy_templates: - name: initial_interval type: text title: Initial Interval - description: Initial interval at which the logs will be pulled. Supported units for this parameter are h/m/s. + description: How much historical data to fetch. Supported units for this parameter are h/m/s. default: 720h multi: false required: true @@ -69,8 +69,8 @@ policy_templates: - name: interval type: text title: Interval - description: Interval at which the logs will be pulled. Recommended value between 10sec to 1 min. Supported units for this parameter are h/m/s. - default: 20s + description: Interval at which to check for new data. Supported units for this parameter are h/m/s. + default: 5m multi: false required: true show_user: true @@ -85,7 +85,7 @@ policy_templates: - name: page_limit type: integer title: Number of results per page - description: Sets the number of results to return per page. Maximum allowed value 1000. + description: Sets the number of results to return per page. Maximum allowed value 1000. Minimum allowed value 10. default: 1000 multi: false required: true