[windows] Defender - Fix pipeline for Path and add missing fields. (#…

…11529) * Fix pipeline for Path and add missing fields. * change log PR update * Add windows_defender.evidence paths for multiple hits, add anotehr test case, generate new expected and README
elastic · Nov 20, 2024 · 792f10a · 792f10a
1 parent e11bdc9
commit 792f10a
Show file tree

Hide file tree

Showing 9 changed files with 747 additions and 29 deletions.
diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.1"
+  changes:
+    - description: Fix improper parsing of Path for Windows Defender, add more winlog fields
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11529
 - version: "2.3.0"
   changes:
     - description: Deprecate third-party REST API import option.

diff --git a/.../data_stream/windows_defender/_dev/test/pipeline/test-events-windows-defender-events.json b/.../data_stream/windows_defender/_dev/test/pipeline/test-events-windows-defender-events.json
diff --git a/...indows_defender/_dev/test/pipeline/test-events-windows-defender-events.json-expected.json b/...indows_defender/_dev/test/pipeline/test-events-windows-defender-events.json-expected.json
diff --git a/packages/windows/data_stream/windows_defender/elasticsearch/ingest_pipeline/default.yml b/packages/windows/data_stream/windows_defender/elasticsearch/ingest_pipeline/default.yml
@@ -157,17 +157,37 @@ processors:
 
     ## File Fields
 
+  - grok:
+      field: winlog.event_data.Path
+      patterns:
+        - "file:_(?<file.path>[^;]+)(; process:_pid:%{NUMBER:process.pid})?"
+      ignore_missing: true
+  - split:
+      field: winlog.event_data.Path
+      separator: ";"
+      target_field: windows_defender.evidence_paths
+      ignore_missing: true
+      ignore_failure: true
+  - trim:
+      field: windows_defender.evidence_paths
+      ignore_missing: true
+      ignore_failure: true
   - gsub:
-      field: "winlog.event_data.Path"
-      pattern: ".*:_"
+      field: windows_defender.evidence_paths
+      pattern: "file:_"
       replacement: ""
-      target_field: "file.path"
       ignore_missing: true
+      ignore_failure: true
   - gsub:
-      field: "winlog.event_data.FileName"
-      pattern: ".*:_"
+      field: windows_defender.evidence_paths
+      pattern: "process:_"
       replacement: ""
-      target_field: "file.path"
+      ignore_missing: true
+      ignore_failure: true
+  - grok:
+      field: winlog.event_data.FileName
+      patterns:
+        - 'file:_(?<file.path>[^;]+)(; process:_pid:%{NUMBER:process.pid})?'
       ignore_missing: true
   - set:
       field: "file.path"

diff --git a/packages/windows/data_stream/windows_defender/fields/fields.yml b/packages/windows/data_stream/windows_defender/fields/fields.yml
@@ -0,0 +1,7 @@
+- name: windows_defender
+  type: group
+  description: All custom fields that are specific to a Windows Defender event will be grouped in this field name.
+  fields:
+    - name: evidence_paths
+      type: keyword
+      description: One or more paths found in the event.
diff --git a/packages/windows/data_stream/windows_defender/fields/winlog.yml b/packages/windows/data_stream/windows_defender/fields/winlog.yml
@@ -308,6 +308,102 @@
           type: keyword
         - name: RTP_state
           type: keyword
+        - name: Action_ID
+          type: keyword
+        - name: Action_Name
+          type: keyword
+        - name: Additional_Actions_ID
+          type: keyword
+        - name: Additional_Actions_String
+          type: keyword
+        - name: Category_ID
+          type: keyword
+        - name: Category_Name
+          type: keyword
+        - name: Current_Engine_Version
+          type: keyword
+        - name: Current_security_intelligence_Version
+          type: keyword
+        - name: Detection_ID
+          type: keyword
+        - name: Detection_Time
+          type: date
+        - name: Detection_User
+          type: keyword
+        - name: Domain
+          type: keyword
+        - name: Engine_Version
+          type: keyword
+        - name: Error_Code
+          type: keyword
+        - name: Error_Description
+          type: keyword
+        - name: Execution_ID
+          type: keyword
+        - name: Execution_Name
+          type: keyword
+        - name: FWLink
+          type: keyword
+        - name: Origin_ID
+          type: keyword
+        - name: Origin_Name
+          type: keyword
+        - name: Post_Clean_Status
+          type: keyword
+        - name: Pre_Execution_Status
+          type: keyword
+        - name: Previous_Engine_Version
+          type: keyword
+        - name: Previous_security_intelligence_Version
+          type: keyword
+        - name: Product_Version
+          type: keyword
+        - name: Remediation_User
+          type: keyword
+        - name: SID
+          type: keyword
+        - name: Scan_ID
+          type: keyword
+        - name: Scan_Parameters
+          type: keyword
+        - name: Scan_Parameters_Index
+          type: keyword
+        - name: Scan_Type
+          type: keyword
+        - name: Scan_Type_Index
+          type: keyword
+        - name: Security_intelligence_Type
+          type: keyword
+        - name: Security_intelligence_Type_Index
+          type: keyword
+        - name: Security_intelligence_Version
+          type: keyword
+        - name: Security_intelligence_version
+          type: keyword
+        - name: Severity_ID
+          type: keyword
+        - name: Severity_Name
+          type: keyword
+        - name: Source_ID
+          type: keyword
+        - name: Source_Name
+          type: keyword
+        - name: Status_Code
+          type: keyword
+        - name: Threat_ID
+          type: keyword
+        - name: Threat_Name
+          type: keyword
+        - name: Type_ID
+          type: keyword
+        - name: Type_Name
+          type: keyword
+        - name: Update_Type
+          type: keyword
+        - name: Update_Type_Index
+          type: keyword
+        - name: User
+          type: keyword
 
     - name: event_id
       type: keyword

diff --git a/packages/windows/data_stream/windows_defender/sample_event.json b/packages/windows/data_stream/windows_defender/sample_event.json
@@ -1,24 +1,24 @@
 {
     "@timestamp": "2024-09-25T19:30:20.339Z",
     "agent": {
-        "ephemeral_id": "8b5286b8-9d5e-4a19-921b-48b6cdf2881d",
-        "id": "1f365586-06e0-4a4b-8787-c5c088f44de5",
-        "name": "elastic-agent-28855",
+        "ephemeral_id": "e9af23ec-c024-4b56-a624-39e242319c16",
+        "id": "4a0bc7fa-6bfd-41c2-9cb6-17a1560abba7",
+        "name": "elastic-agent-41982",
         "type": "filebeat",
-        "version": "8.15.1"
+        "version": "8.15.2"
     },
     "data_stream": {
         "dataset": "windows.windows_defender",
-        "namespace": "39980",
+        "namespace": "97455",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "1f365586-06e0-4a4b-8787-c5c088f44de5",
+        "id": "4a0bc7fa-6bfd-41c2-9cb6-17a1560abba7",
         "snapshot": false,
-        "version": "8.15.1"
+        "version": "8.15.2"
     },
     "event": {
         "action": "malware-quarantined",
@@ -27,9 +27,9 @@
             "malware"
         ],
         "code": "1117",
-        "created": "2024-09-26T15:59:35.718Z",
+        "created": "2024-11-04T23:00:42.213Z",
         "dataset": "windows.windows_defender",
-        "ingested": "2024-09-26T15:59:38Z",
+        "ingested": "2024-11-04T23:00:45Z",
         "kind": "event",
         "original": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Windows Defender' Guid='{11cd958a-c507-4ef3-b3f2-5fd9dfbd2c78}'/><EventID>1117</EventID><Version>0</Version><Level>4</Level><Task>0</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime='2024-09-25T19:30:20.3397185Z'/><EventRecordID>22399</EventRecordID><Correlation ActivityID='{e8e94442-2856-4bab-a775-454654f7ec59}'/><Execution ProcessID='3168' ThreadID='13904'/><Channel>Microsoft-Windows-Windows Defender/Operational</Channel><Computer>el33t-b00k-1.org.local</Computer><Security UserID='S-1-5-18'/></System><EventData><Data Name='Product Name'>Microsoft Defender Antivirus</Data><Data Name='Product Version'>4.18.24080.9</Data><Data Name='Detection ID'>{4E4D1D41-19CC-4EE2-BDB0-950A07B81378}</Data><Data Name='Detection Time'>2024-09-25T19:29:38.198Z</Data><Data Name='Unused'></Data><Data Name='Unused2'></Data><Data Name='Threat ID'>2147680291</Data><Data Name='Threat Name'>Trojan:Win32/Detplock</Data><Data Name='Severity ID'>5</Data><Data Name='Severity Name'>Severe</Data><Data Name='Category ID'>8</Data><Data Name='Category Name'>Trojan</Data><Data Name='FWLink'>https://go.microsoft.com/fwlink/?linkid=37020&amp;name=Trojan:Win32/Detplock&amp;threatid=2147680291&amp;enterprise=1</Data><Data Name='Status Code'>3</Data><Data Name='Status Description'></Data><Data Name='State'>2</Data><Data Name='Source ID'>3</Data><Data Name='Source Name'>Real-Time Protection</Data><Data Name='Process Name'>C:\\Program Files\\Notepad++\\notepad++.exe</Data><Data Name='Detection User'>ORG\\Topsy</Data><Data Name='Unused3'></Data><Data Name='Path'>file:_C:\\Users\\Topsy\\Desktop\\eat_dem_yams.exe</Data><Data Name='Origin ID'>1</Data><Data Name='Origin Name'>Local machine</Data><Data Name='Execution ID'>1</Data><Data Name='Execution Name'>Suspended</Data><Data Name='Type ID'>8</Data><Data Name='Type Name'>FastPath</Data><Data Name='Pre Execution Status'>0</Data><Data Name='Action ID'>2</Data><Data Name='Action Name'>Quarantine</Data><Data Name='Unused4'></Data><Data Name='Error Code'>0x00000000</Data><Data Name='Error Description'>The operation completed successfully. </Data><Data Name='Unused5'></Data><Data Name='Post Clean Status'>0</Data><Data Name='Additional Actions ID'>0</Data><Data Name='Additional Actions String'>No additional actions required</Data><Data Name='Remediation User'>NT AUTHORITY\\SYSTEM</Data><Data Name='Unused6'></Data><Data Name='Security intelligence Version'>AV: 1.419.163.0, AS: 1.419.163.0, NIS: 1.419.163.0</Data><Data Name='Engine Version'>AM: 1.1.24080.9, NIS: 1.1.24080.9</Data></EventData><RenderingInfo Culture='en-US'><Message>Microsoft Defender Antivirus has taken action to protect this machine from malware or other potentially unwanted software.&#13;&#10; For more information please see the following:&#13;&#10;https://go.microsoft.com/fwlink/?linkid=37020&amp;name=Trojan:Win32/Detplock&amp;threatid=2147680291&amp;enterprise=1&#13;&#10; &#9;Name: Trojan:Win32/Detplock&#13;&#10; &#9;ID: 2147680291&#13;&#10; &#9;Severity: Severe&#13;&#10; &#9;Category: Trojan&#13;&#10; &#9;Path: file:_C:\\Users\\Topsy\\Desktop\\eat_dem_yams.exe&#13;&#10; &#9;Detection Origin: Local machine&#13;&#10; &#9;Detection Type: FastPath&#13;&#10; &#9;Detection Source: Real-Time Protection&#13;&#10; &#9;User: NT AUTHORITY\\SYSTEM&#13;&#10; &#9;Process Name: C:\\Program Files\\Notepad++\\notepad++.exe&#13;&#10; &#9;Action: Quarantine&#13;&#10; &#9;Action Status:  No additional actions required&#13;&#10; &#9;Error Code: 0x00000000&#13;&#10; &#9;Error description: The operation completed successfully. &#13;&#10; &#9;Security intelligence Version: AV: 1.419.163.0, AS: 1.419.163.0, NIS: 1.419.163.0&#13;&#10; &#9;Engine Version: AM: 1.1.24080.9, NIS: 1.1.24080.9</Message><Level>Information</Level><Opcode>Info</Opcode><Provider>Microsoft-Windows-Windows Defender</Provider></RenderingInfo></Event>",
         "outcome": "success",
@@ -66,6 +66,11 @@
         "domain": "ORG",
         "name": "Topsy"
     },
+    "windows_defender": {
+        "evidence_paths": [
+            "C:\\Users\\Topsy\\Desktop\\eat_dem_yams.exe"
+        ]
+    },
     "winlog": {
         "activity_id": "{e8e94442-2856-4bab-a775-454654f7ec59}",
         "channel": "Microsoft-Windows-Windows Defender/Operational",