diff --git a/packages/qualys_vmdr/changelog.yml b/packages/qualys_vmdr/changelog.yml index 289d9edaa61..f5fab85f3f0 100644 --- a/packages/qualys_vmdr/changelog.yml +++ b/packages/qualys_vmdr/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "4.2.0" + changes: + - description: Map cloud provider metadata to cloud fields. + type: enhancement + link: https://github.com/elastic/integrations/pull/10605 - version: "4.1.1" changes: - description: Fix handling of the activity_log API response body. diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log b/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log index f738c4e6b3c..5c4ca8a38df 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log @@ -5,3 +5,6 @@ {"NETBIOS": "EXCHB10","IP": "81.2.69.192","ID": "11700976","LAST_PC_SCANNED_DATE": "2023-05-31T11:30:20Z","ASSET_ID": "27858031", "LAST_VM_SCANNED_DATE": "2023-05-31T12:34:44Z","LAST_VM_SCANNED_DURATION": "1822","DNS": "abc10.fdgshb10.local","DNS_DATA": {"DOMAIN": "abc10.local","FQDN": "abc10.fdgshb10.local","HOSTNAME": "abc10"},"LAST_SCAN_DATETIME": "2023-05-31T12:55:28Z","LAST_VM_AUTH_SCANNED_DURATION": "1822", "OS": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607","TRACKING_METHOD": "IP","LAST_VM_AUTH_SCANNED_DATE": "2023-05-31T12:34:44Z"} {"ASSET_ID":"27703780","DETECTION_LIST":{"FIRST_FOUND_DATETIME":"2023-05-30T11:49:24Z","IS_DISABLED":"0","LAST_FOUND_DATETIME":"2023-06-17T12:47:54Z","LAST_PROCESSED_DATETIME":"2023-06-17T13:20:12Z","QID":"70028","RESULTS":"User Name\t(none)\nDomain\t(none)\nAuthentication Scheme\tNULL session\nSecurity\tUser-based\nSMBv1 Signing\tDisabled\nDiscovery Method\tUnable to log in using credentials provided by user, fallback to NULL session\nCIFS Signing\tdefault","SEVERITY":"1","TIMES_FOUND":"38","TYPE":"Info"},"DNS":"win-d24ck5nn676.ldap.local","DNS_DATA":{"DOMAIN":"ldap.local","FQDN":"win-d24ck5nn676.ldap.local","HOSTNAME":"win-d24ck5nn676"},"ID":"11701931","IP":"10.50.2.122","LAST_PC_SCANNED_DATE":"2023-06-18T04:00:17Z","LAST_SCAN_DATETIME":"2023-06-17T13:20:12Z","LAST_VM_SCANNED_DATE":"2023-06-17T12:47:54Z","LAST_VM_SCANNED_DURATION":"1806","NETBIOS":"WIN-D24CK5NN676","OS":"Windows 2016","TAGS":{"TAG":{"NAME":"Windows","TAG_ID":"19429857"}},"TRACKING_METHOD":"IP"} {"NETBIOS": "EXCHB10","NETWORK_ID": 0,"EC2_INSTANCE_ID": "i-07f91cxxx3axxxb3f","CLOUD_RESOURCE_ID": "i-07f91cxxx3axxxb3f","CLOUD_SERVICE": "EC2","CLOUD_PROVIDER": "AWS","QG_HOSTID": "44e2cf13-xxxx-48b9-xxxx-de489547754d","METADATA": {"EC2": {"ATTRIBUTE": {"NAME": "firstDiscovered","LAST_STATUS": "Success","VALUE": "1683517315000","LAST_SUCCESS_DATE": "2023-06-22T12:44:21Z","LAST_ERROR_DATE": "2023-05-22T02:09:49Z","LAST_ERROR": "404 - Not Found"}},"GOOGLE": {"ATTRIBUTE": {"NAME": "firstDiscovered","LAST_STATUS": "Success","VALUE": "1683517315000","LAST_SUCCESS_DATE": "2023-06-22T12:44:21Z","LAST_ERROR_DATE": "2023-05-22T02:09:49Z","LAST_ERROR": "404 - Not Found"}},"AZURE": {"ATTRIBUTE": {"NAME": "firstDiscovered","LAST_STATUS": "Success","VALUE": "1683517315000","LAST_SUCCESS_DATE": "2023-06-22T12:44:21Z","LAST_ERROR_DATE": "2023-05-22T02:09:49Z","LAST_ERROR": "404 - Not Found"}}},"CLOUD_PROVIDER_TAGS": {"CLOUD_TAG": {"NAME": "Name","VALUE": "allocator-dnt-frozen-i3en-2xl-v1-a","LAST_SUCCESS_DATE": "2023-06-22T12:44:21Z"}}, "IP": "81.2.69.192","ID": "11700976","LAST_PC_SCANNED_DATE": "2023-05-31T11:30:20Z","ASSET_ID": "27858031","TAGS": {"TAG": [{"NAME": "Sales","TAG_ID": "19427596"},{"TAG_ID": "19429855","NAME": "Linux"}]},"LAST_VM_SCANNED_DATE": "2023-05-31T12:34:44Z","LAST_VM_SCANNED_DURATION": "1822","DNS": "abc10.fdgshb10.local","DNS_DATA": {"DOMAIN": "abc10.local","FQDN": "abc10.fdgshb10.local","HOSTNAME": "abc10"},"LAST_SCAN_DATETIME": "2023-05-31T12:55:28Z","LAST_VM_AUTH_SCANNED_DURATION": "1822","DETECTION_LIST": {"LAST_UPDATE_DATETIME": "2023-05-30T07:48:14Z","STATUS": "New","FIRST_FOUND_DATETIME": "2023-05-30T07:46:15Z","QID": "11827","SSL": "0","IS_IGNORED": "0","PORT": "443","SEVERITY": "2","LAST_FOUND_DATETIME": "2023-05-30T07:46:15Z","TYPE": "Confirmed","QDS": {"#text": "50","severity": "MEDIUM"},"QDS_FACTORS": {"QDS_FACTOR": [{"#text": "Easy_Exploit,No_Patch","name": "RTI"},{"#text": "5.0","name": "CVSS"}]},"LAST_PROCESSED_DATETIME": "2023-05-30T07:48:14Z","PROTOCOL": "tcp","TIMES_FOUND": "1","IS_DISABLED": "1","RESULTS": "X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.","LAST_TEST_DATETIME": "2023-05-30T07:46:15Z"}, "OS": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607","TRACKING_METHOD": "IP","LAST_VM_AUTH_SCANNED_DATE": "2023-05-31T12:34:44Z"} +{"NETBIOS":"EXCHB10","NETWORK_ID":0,"IPV6":"0.0.0.0","OS_CPE":"xyz","EC2_INSTANCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_RESOURCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_SERVICE":"EC2","CLOUD_PROVIDER":"AWS","QG_HOSTID":"44e2cf13-xxxx-48b9-xxxx-de489547754d","METADATA":{"EC2":{"ATTRIBUTE":[{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T13:50:30Z","NAME":"latest/dynamic/instance-identity/document/accountId","VALUE":"123456789123"},{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T13:50:30Z","NAME":"latest/dynamic/instance-identity/document/availabilityZone","VALUE":"us-west-2b"}]}},"CLOUD_PROVIDER_TAGS":{"CLOUD_TAG":[{"NAME":"Name","VALUE":"allocator-dnt-frozen-i3en-2xl-v1-a","LAST_SUCCESS_DATE":"2023-06-22T12:44:21Z"}]},"IP":"81.2.69.192","ID":"11700976","LAST_PC_SCANNED_DATE":"2023-05-31T11:30:20Z","ASSET_ID":"27858031","TAGS":{"TAG":[{"NAME":"Sales","TAG_ID":"19427596","COLOR":"#FFFFF","BACKGROUND_COLOR":"#FFFFF"},{"TAG_ID":"19429855","NAME":"Linux"}]},"LAST_VM_SCANNED_DATE":"2023-05-31T12:34:44Z","LAST_VM_SCANNED_DURATION":"1822","DNS":"abc10.fdgshb10.local","DNS_DATA":{"DOMAIN":"abc10.local","FQDN":"abc10.fdgshb10.local","HOSTNAME":"abc10"},"LAST_SCAN_DATETIME":"2023-05-31T12:55:28Z","LAST_VM_AUTH_SCANNED_DURATION":"1822","DETECTION_LIST":{"LAST_UPDATE_DATETIME":"2023-05-30T07:48:14Z","LAST_FIXED_DATETIME":"2023-05-22T02:09:49Z","FIRST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","LAST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","TIMES_REOPENED":"2","SERVICE":"service1","AFFECT_RUNNING_KERNEL":"kernel1","AFFECT_RUNNING_SERVICE":"service1","AFFECT_EXPLOITABLE_CONFIG":"config1","ASSET_CVE":"cve3","STATUS":"New","FQDN":"exchb10.exchb10.local","INSTANCE":"instance1","FIRST_FOUND_DATETIME":"2023-05-30T07:46:15Z","QID":"11827","SSL":"0","IS_IGNORED":"0","PORT":"443","SEVERITY":"2","LAST_FOUND_DATETIME":"2023-05-30T07:46:15Z","TYPE":"Confirmed","QDS":{"#text":"50","severity":"MEDIUM"},"QDS_FACTORS":{"QDS_FACTOR":[{"#text":"Easy_Exploit,No_Patch","name":"RTI"},{"#text":"5.0","name":"CVSS"}]},"LAST_PROCESSED_DATETIME":"2023-05-30T07:48:14Z","PROTOCOL":"tcp","TIMES_FOUND":"1","IS_DISABLED":"1","RESULTS":"X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.","LAST_TEST_DATETIME":"2023-05-30T07:46:15Z"},"OS":"Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607","TRACKING_METHOD":"IP","LAST_VM_AUTH_SCANNED_DATE":"2023-05-31T12:34:44Z"} +{"NETBIOS":"EXCHB10","NETWORK_ID":0,"IPV6":"0.0.0.0","OS_CPE":"xyz","EC2_INSTANCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_RESOURCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_SERVICE":"GCP","CLOUD_PROVIDER":"Google","QG_HOSTID":"44e2cf13-xxxx-48b9-xxxx-de489547754d","METADATA":{"GOOGLE":{"ATTRIBUTE":[{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T12:05:04Z","NAME":"machineType","VALUE":"custom-2-3072"},{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T12:05:04Z","NAME":"projectIdNo","VALUE":"123456789123"}]}},"CLOUD_PROVIDER_TAGS":{"CLOUD_TAG":[{"NAME":"Name","VALUE":"allocator-dnt-frozen-i3en-2xl-v1-a","LAST_SUCCESS_DATE":"2023-06-22T12:44:21Z"}]},"IP":"81.2.69.192","ID":"11700976","LAST_PC_SCANNED_DATE":"2023-05-31T11:30:20Z","ASSET_ID":"27858031","TAGS":{"TAG":[{"NAME":"Sales","TAG_ID":"19427596","COLOR":"#FFFFF","BACKGROUND_COLOR":"#FFFFF"},{"TAG_ID":"19429855","NAME":"Linux"}]},"LAST_VM_SCANNED_DATE":"2023-05-31T12:34:44Z","LAST_VM_SCANNED_DURATION":"1822","DNS":"abc10.fdgshb10.local","DNS_DATA":{"DOMAIN":"abc10.local","FQDN":"abc10.fdgshb10.local","HOSTNAME":"abc10"},"LAST_SCAN_DATETIME":"2023-05-31T12:55:28Z","LAST_VM_AUTH_SCANNED_DURATION":"1822","DETECTION_LIST":{"LAST_UPDATE_DATETIME":"2023-05-30T07:48:14Z","LAST_FIXED_DATETIME":"2023-05-22T02:09:49Z","FIRST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","LAST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","TIMES_REOPENED":"2","SERVICE":"service1","AFFECT_RUNNING_KERNEL":"kernel1","AFFECT_RUNNING_SERVICE":"service1","AFFECT_EXPLOITABLE_CONFIG":"config1","ASSET_CVE":"cve3","STATUS":"New","FQDN":"exchb10.exchb10.local","INSTANCE":"instance1","FIRST_FOUND_DATETIME":"2023-05-30T07:46:15Z","QID":"11827","SSL":"0","IS_IGNORED":"0","PORT":"443","SEVERITY":"2","LAST_FOUND_DATETIME":"2023-05-30T07:46:15Z","TYPE":"Confirmed","QDS":{"#text":"50","severity":"MEDIUM"},"QDS_FACTORS":{"QDS_FACTOR":[{"#text":"Easy_Exploit,No_Patch","name":"RTI"},{"#text":"5.0","name":"CVSS"}]},"LAST_PROCESSED_DATETIME":"2023-05-30T07:48:14Z","PROTOCOL":"tcp","TIMES_FOUND":"1","IS_DISABLED":"1","RESULTS":"X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.","LAST_TEST_DATETIME":"2023-05-30T07:46:15Z"},"OS":"Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607","TRACKING_METHOD":"IP","LAST_VM_AUTH_SCANNED_DATE":"2023-05-31T12:34:44Z"} +{"NETBIOS":"EXCHB10","NETWORK_ID":0,"IPV6":"0.0.0.0","OS_CPE":"xyz","EC2_INSTANCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_RESOURCE_ID":"i-07f91cxxx3axxxb3f","CLOUD_SERVICE":"Azure","CLOUD_PROVIDER":"Azure","QG_HOSTID":"44e2cf13-xxxx-48b9-xxxx-de489547754d","METADATA":{"AZURE":{"ATTRIBUTE":[{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T12:24:57Z","NAME":"location","VALUE":"eastus2"},{"LAST_ERROR":"","LAST_ERROR_DATE":"","LAST_STATUS":"Success","LAST_SUCCESS_DATE":"2024-07-25T12:24:57Z","NAME":"subscriptionId","VALUE":"00000000-0000-0000-0000-000000000000"}]}},"CLOUD_PROVIDER_TAGS":{"CLOUD_TAG":[{"NAME":"Name","VALUE":"allocator-dnt-frozen-i3en-2xl-v1-a","LAST_SUCCESS_DATE":"2023-06-22T12:44:21Z"}]},"IP":"81.2.69.192","ID":"11700976","LAST_PC_SCANNED_DATE":"2023-05-31T11:30:20Z","ASSET_ID":"27858031","TAGS":{"TAG":[{"NAME":"Sales","TAG_ID":"19427596","COLOR":"#FFFFF","BACKGROUND_COLOR":"#FFFFF"},{"TAG_ID":"19429855","NAME":"Linux"}]},"LAST_VM_SCANNED_DATE":"2023-05-31T12:34:44Z","LAST_VM_SCANNED_DURATION":"1822","DNS":"abc10.fdgshb10.local","DNS_DATA":{"DOMAIN":"abc10.local","FQDN":"abc10.fdgshb10.local","HOSTNAME":"abc10"},"LAST_SCAN_DATETIME":"2023-05-31T12:55:28Z","LAST_VM_AUTH_SCANNED_DURATION":"1822","DETECTION_LIST":{"LAST_UPDATE_DATETIME":"2023-05-30T07:48:14Z","LAST_FIXED_DATETIME":"2023-05-22T02:09:49Z","FIRST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","LAST_REOPENED_DATETIME":"2023-05-22T02:09:49Z","TIMES_REOPENED":"2","SERVICE":"service1","AFFECT_RUNNING_KERNEL":"kernel1","AFFECT_RUNNING_SERVICE":"service1","AFFECT_EXPLOITABLE_CONFIG":"config1","ASSET_CVE":"cve3","STATUS":"New","FQDN":"exchb10.exchb10.local","INSTANCE":"instance1","FIRST_FOUND_DATETIME":"2023-05-30T07:46:15Z","QID":"11827","SSL":"0","IS_IGNORED":"0","PORT":"443","SEVERITY":"2","LAST_FOUND_DATETIME":"2023-05-30T07:46:15Z","TYPE":"Confirmed","QDS":{"#text":"50","severity":"MEDIUM"},"QDS_FACTORS":{"QDS_FACTOR":[{"#text":"Easy_Exploit,No_Patch","name":"RTI"},{"#text":"5.0","name":"CVSS"}]},"LAST_PROCESSED_DATETIME":"2023-05-30T07:48:14Z","PROTOCOL":"tcp","TIMES_FOUND":"1","IS_DISABLED":"1","RESULTS":"X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.","LAST_TEST_DATETIME":"2023-05-30T07:46:15Z"},"OS":"Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607","TRACKING_METHOD":"IP","LAST_VM_AUTH_SCANNED_DATE":"2023-05-31T12:34:44Z"} diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log-expected.json b/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log-expected.json index f9dd74743a6..478a73a92c2 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log-expected.json +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/_dev/test/pipeline/test-asset-host-detection.log-expected.json @@ -3,7 +3,8 @@ { "cloud": { "instance": { - "id": "i-07f91cxxx3axxxb3f" + "id": "i-07f91cxxx3axxxb3f", + "name": "abc10" }, "provider": "AWS", "service": { @@ -205,6 +206,11 @@ ] }, { + "cloud": { + "instance": { + "name": "abc10" + } + }, "ecs": { "version": "8.11.0" }, @@ -306,6 +312,11 @@ ] }, { + "cloud": { + "instance": { + "name": "abc10" + } + }, "ecs": { "version": "8.11.0" }, @@ -414,6 +425,11 @@ ] }, { + "cloud": { + "instance": { + "name": "abc10" + } + }, "ecs": { "version": "8.11.0" }, @@ -516,6 +532,11 @@ ] }, { + "cloud": { + "instance": { + "name": "abc10" + } + }, "ecs": { "version": "8.11.0" }, @@ -576,6 +597,11 @@ ] }, { + "cloud": { + "instance": { + "name": "win-d24ck5nn676" + } + }, "ecs": { "version": "8.11.0" }, @@ -657,7 +683,8 @@ { "cloud": { "instance": { - "id": "i-07f91cxxx3axxxb3f" + "id": "i-07f91cxxx3axxxb3f", + "name": "abc10" }, "provider": "AWS", "service": { @@ -828,6 +855,563 @@ "tags": [ "preserve_duplicate_custom_fields" ] + }, + { + "cloud": { + "account": { + "id": [ + "123456789123" + ] + }, + "availability_zone": [ + "us-west-2b" + ], + "instance": { + "id": "i-07f91cxxx3axxxb3f", + "name": "abc10" + }, + "provider": "AWS", + "service": { + "name": "EC2" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "host" + ], + "kind": "alert", + "type": [ + "info" + ] + }, + "host": { + "domain": "EXCHB10", + "id": "11700976", + "ip": [ + "81.2.69.192" + ], + "os": { + "full": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607" + } + }, + "qualys_vmdr": { + "asset_host_detection": { + "asset_id": "27858031", + "cloud_provider": "AWS", + "cloud_provider_tags": { + "cloud_tag": [ + { + "last_success_date": "2023-06-22T12:44:21.000Z", + "name": "Name", + "value": "allocator-dnt-frozen-i3en-2xl-v1-a" + } + ] + }, + "cloud_resource_id": "i-07f91cxxx3axxxb3f", + "cloud_service": "EC2", + "dns": "abc10.fdgshb10.local", + "dns_data": { + "domain": "abc10.local", + "fqdn": "abc10.fdgshb10.local", + "hostname": "abc10" + }, + "ec2_instance_id": "i-07f91cxxx3axxxb3f", + "id": "11700976", + "ip": "81.2.69.192", + "ipv6": "0.0.0.0", + "last_pc_scanned_date": "2023-05-31T11:30:20.000Z", + "last_scan_datetime": "2023-05-31T12:55:28.000Z", + "last_vm_auth_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_auth_scanned_duration": 1822, + "last_vm_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_scanned_duration": 1822, + "metadata": { + "ec2": { + "attribute": [ + { + "last": { + "status": "Success", + "success_date": "2024-07-25T13:50:30.000Z" + }, + "name": "latest/dynamic/instance-identity/document/accountId", + "value": "123456789123" + }, + { + "last": { + "status": "Success", + "success_date": "2024-07-25T13:50:30.000Z" + }, + "name": "latest/dynamic/instance-identity/document/availabilityZone", + "value": "us-west-2b" + } + ] + } + }, + "netbios": "EXCHB10", + "network_id": "0", + "os": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607", + "os_cpe": "xyz", + "qg_hostid": "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "tags": [ + { + "background_color": "#FFFFF", + "color": "#FFFFF", + "id": "19427596", + "name": "Sales" + }, + { + "id": "19429855", + "name": "Linux" + } + ], + "tracking_method": "IP", + "vulnerability": { + "affect": { + "exploitable_config": "config1", + "running": { + "kernel": "kernel1", + "service": "service1" + } + }, + "asset_cve": "cve3", + "first": { + "found_datetime": "2023-05-30T07:46:15.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z" + }, + "fqdn": "exchb10.exchb10.local", + "instance": "instance1", + "is_disabled": true, + "is_ignored": false, + "last": { + "fixed_datetime": "2023-05-22T02:09:49.000Z", + "found_datetime": "2023-05-30T07:46:15.000Z", + "processed_datetime": "2023-05-30T07:48:14.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z", + "test_datetime": "2023-05-30T07:46:15.000Z", + "update_datetime": "2023-05-30T07:48:14.000Z" + }, + "port": 443, + "protocol": "tcp", + "qds": { + "severity": "MEDIUM", + "text": "50" + }, + "qds_factors": [ + { + "name": "RTI", + "text": "Easy_Exploit,No_Patch" + }, + { + "name": "CVSS", + "text": "5.0" + } + ], + "qid": "11827", + "results": "X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.", + "service": "service1", + "severity": 2, + "ssl": "0", + "status": "New", + "times": { + "found": 1, + "reopened": 2 + }, + "type": "Confirmed" + } + } + }, + "related": { + "hosts": [ + "abc10.fdgshb10.local", + "abc10", + "11700976", + "EXCHB10", + "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "exchb10.exchb10.local" + ], + "ip": [ + "81.2.69.192", + "0.0.0.0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ] + }, + { + "cloud": { + "instance": { + "id": "i-07f91cxxx3axxxb3f", + "name": "abc10" + }, + "machine": { + "type": [ + "custom-2-3072" + ] + }, + "project": { + "id": [ + "123456789123" + ] + }, + "provider": "Google", + "service": { + "name": "GCP" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "host" + ], + "kind": "alert", + "type": [ + "info" + ] + }, + "host": { + "domain": "EXCHB10", + "id": "11700976", + "ip": [ + "81.2.69.192" + ], + "os": { + "full": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607" + } + }, + "qualys_vmdr": { + "asset_host_detection": { + "asset_id": "27858031", + "cloud_provider": "Google", + "cloud_provider_tags": { + "cloud_tag": [ + { + "last_success_date": "2023-06-22T12:44:21.000Z", + "name": "Name", + "value": "allocator-dnt-frozen-i3en-2xl-v1-a" + } + ] + }, + "cloud_resource_id": "i-07f91cxxx3axxxb3f", + "cloud_service": "GCP", + "dns": "abc10.fdgshb10.local", + "dns_data": { + "domain": "abc10.local", + "fqdn": "abc10.fdgshb10.local", + "hostname": "abc10" + }, + "ec2_instance_id": "i-07f91cxxx3axxxb3f", + "id": "11700976", + "ip": "81.2.69.192", + "ipv6": "0.0.0.0", + "last_pc_scanned_date": "2023-05-31T11:30:20.000Z", + "last_scan_datetime": "2023-05-31T12:55:28.000Z", + "last_vm_auth_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_auth_scanned_duration": 1822, + "last_vm_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_scanned_duration": 1822, + "metadata": { + "google": { + "attribute": [ + { + "last": { + "status": "Success", + "success_date": "2024-07-25T12:05:04.000Z" + }, + "name": "machineType", + "value": "custom-2-3072" + }, + { + "last": { + "status": "Success", + "success_date": "2024-07-25T12:05:04.000Z" + }, + "name": "projectIdNo", + "value": "123456789123" + } + ] + } + }, + "netbios": "EXCHB10", + "network_id": "0", + "os": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607", + "os_cpe": "xyz", + "qg_hostid": "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "tags": [ + { + "background_color": "#FFFFF", + "color": "#FFFFF", + "id": "19427596", + "name": "Sales" + }, + { + "id": "19429855", + "name": "Linux" + } + ], + "tracking_method": "IP", + "vulnerability": { + "affect": { + "exploitable_config": "config1", + "running": { + "kernel": "kernel1", + "service": "service1" + } + }, + "asset_cve": "cve3", + "first": { + "found_datetime": "2023-05-30T07:46:15.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z" + }, + "fqdn": "exchb10.exchb10.local", + "instance": "instance1", + "is_disabled": true, + "is_ignored": false, + "last": { + "fixed_datetime": "2023-05-22T02:09:49.000Z", + "found_datetime": "2023-05-30T07:46:15.000Z", + "processed_datetime": "2023-05-30T07:48:14.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z", + "test_datetime": "2023-05-30T07:46:15.000Z", + "update_datetime": "2023-05-30T07:48:14.000Z" + }, + "port": 443, + "protocol": "tcp", + "qds": { + "severity": "MEDIUM", + "text": "50" + }, + "qds_factors": [ + { + "name": "RTI", + "text": "Easy_Exploit,No_Patch" + }, + { + "name": "CVSS", + "text": "5.0" + } + ], + "qid": "11827", + "results": "X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.", + "service": "service1", + "severity": 2, + "ssl": "0", + "status": "New", + "times": { + "found": 1, + "reopened": 2 + }, + "type": "Confirmed" + } + } + }, + "related": { + "hosts": [ + "abc10.fdgshb10.local", + "abc10", + "11700976", + "EXCHB10", + "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "exchb10.exchb10.local" + ], + "ip": [ + "81.2.69.192", + "0.0.0.0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ] + }, + { + "cloud": { + "instance": { + "id": "i-07f91cxxx3axxxb3f", + "name": "abc10" + }, + "project": { + "id": [ + "00000000-0000-0000-0000-000000000000" + ] + }, + "provider": "Azure", + "region": [ + "eastus2" + ], + "service": { + "name": "Azure" + } + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "host" + ], + "kind": "alert", + "type": [ + "info" + ] + }, + "host": { + "domain": "EXCHB10", + "id": "11700976", + "ip": [ + "81.2.69.192" + ], + "os": { + "full": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607" + } + }, + "qualys_vmdr": { + "asset_host_detection": { + "asset_id": "27858031", + "cloud_provider": "Azure", + "cloud_provider_tags": { + "cloud_tag": [ + { + "last_success_date": "2023-06-22T12:44:21.000Z", + "name": "Name", + "value": "allocator-dnt-frozen-i3en-2xl-v1-a" + } + ] + }, + "cloud_resource_id": "i-07f91cxxx3axxxb3f", + "cloud_service": "Azure", + "dns": "abc10.fdgshb10.local", + "dns_data": { + "domain": "abc10.local", + "fqdn": "abc10.fdgshb10.local", + "hostname": "abc10" + }, + "ec2_instance_id": "i-07f91cxxx3axxxb3f", + "id": "11700976", + "ip": "81.2.69.192", + "ipv6": "0.0.0.0", + "last_pc_scanned_date": "2023-05-31T11:30:20.000Z", + "last_scan_datetime": "2023-05-31T12:55:28.000Z", + "last_vm_auth_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_auth_scanned_duration": 1822, + "last_vm_scanned_date": "2023-05-31T12:34:44.000Z", + "last_vm_scanned_duration": 1822, + "metadata": { + "azure": { + "attribute": [ + { + "last": { + "status": "Success", + "success_date": "2024-07-25T12:24:57.000Z" + }, + "name": "location", + "value": "eastus2" + }, + { + "last": { + "status": "Success", + "success_date": "2024-07-25T12:24:57.000Z" + }, + "name": "subscriptionId", + "value": "00000000-0000-0000-0000-000000000000" + } + ] + } + }, + "netbios": "EXCHB10", + "network_id": "0", + "os": "Windows Server 2016 Standard Evaluation 64 bit Edition AD Version 1607", + "os_cpe": "xyz", + "qg_hostid": "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "tags": [ + { + "background_color": "#FFFFF", + "color": "#FFFFF", + "id": "19427596", + "name": "Sales" + }, + { + "id": "19429855", + "name": "Linux" + } + ], + "tracking_method": "IP", + "vulnerability": { + "affect": { + "exploitable_config": "config1", + "running": { + "kernel": "kernel1", + "service": "service1" + } + }, + "asset_cve": "cve3", + "first": { + "found_datetime": "2023-05-30T07:46:15.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z" + }, + "fqdn": "exchb10.exchb10.local", + "instance": "instance1", + "is_disabled": true, + "is_ignored": false, + "last": { + "fixed_datetime": "2023-05-22T02:09:49.000Z", + "found_datetime": "2023-05-30T07:46:15.000Z", + "processed_datetime": "2023-05-30T07:48:14.000Z", + "reopened_datetime": "2023-05-22T02:09:49.000Z", + "test_datetime": "2023-05-30T07:46:15.000Z", + "update_datetime": "2023-05-30T07:48:14.000Z" + }, + "port": 443, + "protocol": "tcp", + "qds": { + "severity": "MEDIUM", + "text": "50" + }, + "qds_factors": [ + { + "name": "RTI", + "text": "Easy_Exploit,No_Patch" + }, + { + "name": "CVSS", + "text": "5.0" + } + ], + "qid": "11827", + "results": "X-Content-Type-Options HTTP Header missing on port 443.\n\nGET / HTTP/1.0\nHost: 81.2.69.192\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0\n\n\n\nHTTP/1.1 200 OK\nContent-Type: text/html\nLast-Modified: Fri, 29 Mar 2019 10:51:17 GMT\nAccept-Ranges: bytes\nETag: "135e2b561de6d41:0"\nServer: Microsoft-IIS/10.0\nX-Powered-By: ASP.NET\nDate: Tue, 30 May 2023 05:56:00 GMT\nConnection: keep-alive\nContent-Length: 703\n\nStrict-Transport-Security HTTP Header missing on port 443.", + "service": "service1", + "severity": 2, + "ssl": "0", + "status": "New", + "times": { + "found": 1, + "reopened": 2 + }, + "type": "Confirmed" + } + } + }, + "related": { + "hosts": [ + "abc10.fdgshb10.local", + "abc10", + "11700976", + "EXCHB10", + "44e2cf13-xxxx-48b9-xxxx-de489547754d", + "exchb10.exchb10.local" + ], + "ip": [ + "81.2.69.192", + "0.0.0.0" + ] + }, + "tags": [ + "preserve_duplicate_custom_fields" + ] } ] } \ No newline at end of file diff --git a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml index a1095b08f53..ba0f3350d31 100644 --- a/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml +++ b/packages/qualys_vmdr/data_stream/asset_host_detection/elasticsearch/ingest_pipeline/default.yml @@ -785,6 +785,139 @@ processors: - _ingest._value.LAST_SUCCESS_DATE tag: remove_qualys_vmdr_asset_host_detection_cloud_provider_tags_cloud_tag_fields ignore_missing: true + + - script: + lang: painless + tag: cloud_provider_attributes + if: >- + ctx.qualys_vmdr?.asset_host_detection?.metadata?.ec2?.attribute instanceof List || + ctx.qualys_vmdr?.asset_host_detection?.metadata?.google?.attribute instanceof List || + ctx.qualys_vmdr?.asset_host_detection?.metadata?.azure?.attribute instanceof List + source: |- + def accIDs = new ArrayList(); + def projIDs = new ArrayList(); + def names = new ArrayList(); + def regions = new ArrayList(); + def types = new ArrayList(); + def zones = new ArrayList(); + // Collate attributes. + if (ctx.qualys_vmdr?.asset_host_detection?.metadata?.ec2?.attribute instanceof List) { + for (def attr: ctx.qualys_vmdr.asset_host_detection.metadata.ec2.attribute) { + if (attr.value == null && attr.value == "") { + continue; + } + if (attr.name == "latest/dynamic/instance-identity/document/accountId") { + accIDs.add(attr.value); + continue; + } + if (attr.name == "latest/dynamic/instance-identity/document/instanceType") { + types.add(attr.value); + continue; + } + if (attr.name == "latest/dynamic/instance-identity/document/region") { + regions.add(attr.value); + continue; + } + if (attr.name == "latest/dynamic/instance-identity/document/availabilityZone") { + zones.add(attr.value); + } + } + } + if (ctx.qualys_vmdr?.asset_host_detection?.metadata?.google?.attribute instanceof List) { + for (def attr: ctx.qualys_vmdr.asset_host_detection.metadata.google.attribute) { + if (attr.value == null && attr.value == "") { + continue; + } + if (attr.name == "projectId") { + names.add(attr.value); + continue; + } + if (attr.name == "projectIdNo") { + projIDs.add(attr.value); + continue; + } + if (attr.name == "machineType") { + types.add(attr.value); + continue; + } + if (attr.name == "location") { + regions.add(attr.value); + continue; + } + if (attr.name == "zone") { + zones.add(attr.value); + } + } + } + if (ctx.qualys_vmdr?.asset_host_detection?.metadata?.azure?.attribute instanceof List) { + for (def attr: ctx.qualys_vmdr.asset_host_detection.metadata.azure.attribute) { + if (attr.value == null && attr.value == "") { + continue; + } + if (attr.name == "subscriptionId") { + projIDs.add(attr.value); + continue; + } + if (attr.name == "location") { + regions.add(attr.value); + } + } + } + // Apply collation. + if (accIDs.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + if (ctx.cloud.account == null) { + ctx.cloud.account = new HashMap(); + } + ctx.cloud.account.id = accIDs; + } + if (projIDs.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + if (ctx.cloud.project == null) { + ctx.cloud.project = new HashMap(); + } + ctx.cloud.project.id = projIDs; + } + if (names.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + if (ctx.cloud.project == null) { + ctx.cloud.project = new HashMap(); + } + ctx.cloud.project.name = names; + } + if (regions.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + ctx.cloud.region = regions; + } + if (types.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + if (ctx.cloud.machine == null) { + ctx.cloud.machine = new HashMap(); + } + ctx.cloud.machine.type = types; + } + if (zones.length != 0) { + if (ctx.cloud == null) { + ctx.cloud = new HashMap(); + } + ctx.cloud.availability_zone = zones; + } + + - set: + field: cloud.instance.name + copy_from: qualys_vmdr.asset_host_detection.dns_data.hostname + ignore_empty_value: true + - script: lang: painless tag: script_to_set_IS_DISABLED diff --git a/packages/qualys_vmdr/manifest.yml b/packages/qualys_vmdr/manifest.yml index 95f20309db4..54007a24529 100644 --- a/packages/qualys_vmdr/manifest.yml +++ b/packages/qualys_vmdr/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: qualys_vmdr title: Qualys VMDR -version: "4.1.1" +version: "4.2.0" description: Collect data from Qualys VMDR platform with Elastic Agent. type: integration categories: