[windows] Windows Defender Data stream overhaul to GA (#11249)

* Defender Data stream overhaul to GA
* Adjust pipeline to ensure event type is applied
* Update Readme
* Improve test data with event_data blocks, switch to GSUB and SET for file path extractions.
* Generated new JSON test files

packages/windows/_dev/deploy/docker/files/config.yml

@@ -378,7 +378,7 @@ rules:
                   "_bkt": "main~0~1212176D-89E1-485D-89E6-3ADC276CCA38",
                   "_cd": "0:315",
                   "_indextime": "1622471463",
-                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Windows Defender' Guid='{11cd958a-c507-4ef3-b3f2-5fd9dfbd2c78}'/><EventID>1151</EventID><Version>0</Version><Level>4</Level><Task>0</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime='2024-06-21T07:56:39.3136791Z'/><EventRecordID>5655</EventRecordID><Correlation/><Execution ProcessID='7676' ThreadID='15392'/><Channel>Microsoft-Windows-Windows Defender/Operational</Channel><Computer>el33t-b00k-1</Computer><Security UserID='S-1-5-18'/></System><EventData><Data Name='Product Name'>Microsoft Defender Antivirus</Data><Data Name='Platform version'>4.18.24050.7</Data><Data Name='Unused'></Data><Data Name='Engine version'>1.1.24050.5</Data><Data Name='NRI engine version'>1.1.24050.5</Data><Data Name='AV security intelligence version'>1.413.419.0</Data><Data Name='AS security intelligence version'>1.413.419.0</Data><Data Name='NRI security intelligence version'>1.413.419.0</Data><Data Name='RTP state'>Enabled</Data><Data Name='OA state'>Enabled</Data><Data Name='IOAV state'>Enabled</Data><Data Name='BM state'>Enabled</Data><Data Name='Last AV security intelligence age'>0</Data><Data Name='Last AS security intelligence age'>0</Data><Data Name='Last quick scan age'>1</Data><Data Name='Last full scan age'>4294967295</Data><Data Name='AV security intelligence creation time'>2024-06-20T17:59:45Z</Data><Data Name='AS security intelligence creation time'>2024-06-20T17:59:47Z</Data><Data Name='Last quick scan start time'>2024-06-19T15:49:55Z</Data><Data Name='Last quick scan end time'>2024-06-19T15:51:57Z</Data><Data Name='Last quick scan source'>2</Data><Data Name='Last full scan start time'>1601-01-01T00:00:00Z</Data><Data Name='Last full scan end time'>1601-01-01T00:00:00Z</Data><Data Name='Last full scan source'>0</Data><Data Name='Product status'>0x00080000</Data><Data Name='Latest engine version'>1.1.24050.5</Data><Data Name='Engine up-to-date'>0</Data><Data Name='Latest platform version'>4.18.24050.7</Data><Data Name='Platform up-to-date'>1</Data></EventData></Event>",
+                  "_raw": "<Event xmlns='http://schemas.microsoft.com/win/2004/08/events/event'><System><Provider Name='Microsoft-Windows-Windows Defender' Guid='{11cd958a-c507-4ef3-b3f2-5fd9dfbd2c78}'/><EventID>1117</EventID><Version>0</Version><Level>4</Level><Task>0</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime='2024-09-25T19:30:20.3397185Z'/><EventRecordID>22399</EventRecordID><Correlation ActivityID='{e8e94442-2856-4bab-a775-454654f7ec59}'/><Execution ProcessID='3168' ThreadID='13904'/><Channel>Microsoft-Windows-Windows Defender/Operational</Channel><Computer>el33t-b00k-1.org.local</Computer><Security UserID='S-1-5-18'/></System><EventData><Data Name='Product Name'>Microsoft Defender Antivirus</Data><Data Name='Product Version'>4.18.24080.9</Data><Data Name='Detection ID'>{4E4D1D41-19CC-4EE2-BDB0-950A07B81378}</Data><Data Name='Detection Time'>2024-09-25T19:29:38.198Z</Data><Data Name='Unused'></Data><Data Name='Unused2'></Data><Data Name='Threat ID'>2147680291</Data><Data Name='Threat Name'>Trojan:Win32/Detplock</Data><Data Name='Severity ID'>5</Data><Data Name='Severity Name'>Severe</Data><Data Name='Category ID'>8</Data><Data Name='Category Name'>Trojan</Data><Data Name='FWLink'>https://go.microsoft.com/fwlink/?linkid=37020&amp;name=Trojan:Win32/Detplock&amp;threatid=2147680291&amp;enterprise=1</Data><Data Name='Status Code'>3</Data><Data Name='Status Description'></Data><Data Name='State'>2</Data><Data Name='Source ID'>3</Data><Data Name='Source Name'>Real-Time Protection</Data><Data Name='Process Name'>C:\\Program Files\\Notepad++\\notepad++.exe</Data><Data Name='Detection User'>ORG\\Topsy</Data><Data Name='Unused3'></Data><Data Name='Path'>file:_C:\\Users\\Topsy\\Desktop\\eat_dem_yams.exe</Data><Data Name='Origin ID'>1</Data><Data Name='Origin Name'>Local machine</Data><Data Name='Execution ID'>1</Data><Data Name='Execution Name'>Suspended</Data><Data Name='Type ID'>8</Data><Data Name='Type Name'>FastPath</Data><Data Name='Pre Execution Status'>0</Data><Data Name='Action ID'>2</Data><Data Name='Action Name'>Quarantine</Data><Data Name='Unused4'></Data><Data Name='Error Code'>0x00000000</Data><Data Name='Error Description'>The operation completed successfully. </Data><Data Name='Unused5'></Data><Data Name='Post Clean Status'>0</Data><Data Name='Additional Actions ID'>0</Data><Data Name='Additional Actions String'>No additional actions required</Data><Data Name='Remediation User'>NT AUTHORITY\\SYSTEM</Data><Data Name='Unused6'></Data><Data Name='Security intelligence Version'>AV: 1.419.163.0, AS: 1.419.163.0, NIS: 1.419.163.0</Data><Data Name='Engine Version'>AM: 1.1.24080.9, NIS: 1.1.24080.9</Data></EventData><RenderingInfo Culture='en-US'><Message>Microsoft Defender Antivirus has taken action to protect this machine from malware or other potentially unwanted software.&#13;&#10; For more information please see the following:&#13;&#10;https://go.microsoft.com/fwlink/?linkid=37020&amp;name=Trojan:Win32/Detplock&amp;threatid=2147680291&amp;enterprise=1&#13;&#10; &#9;Name: Trojan:Win32/Detplock&#13;&#10; &#9;ID: 2147680291&#13;&#10; &#9;Severity: Severe&#13;&#10; &#9;Category: Trojan&#13;&#10; &#9;Path: file:_C:\\Users\\Topsy\\Desktop\\eat_dem_yams.exe&#13;&#10; &#9;Detection Origin: Local machine&#13;&#10; &#9;Detection Type: FastPath&#13;&#10; &#9;Detection Source: Real-Time Protection&#13;&#10; &#9;User: NT AUTHORITY\\SYSTEM&#13;&#10; &#9;Process Name: C:\\Program Files\\Notepad++\\notepad++.exe&#13;&#10; &#9;Action: Quarantine&#13;&#10; &#9;Action Status:  No additional actions required&#13;&#10; &#9;Error Code: 0x00000000&#13;&#10; &#9;Error description: The operation completed successfully. &#13;&#10; &#9;Security intelligence Version: AV: 1.419.163.0, AS: 1.419.163.0, NIS: 1.419.163.0&#13;&#10; &#9;Engine Version: AM: 1.1.24080.9, NIS: 1.1.24080.9</Message><Level>Information</Level><Opcode>Info</Opcode><Provider>Microsoft-Windows-Windows Defender</Provider></RenderingInfo></Event>",
                   "_serial": "194",
                   "_si": [
                       "69819b6ce1bd",

packages/windows/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.2.0"
+  changes:
+    - description: Improve Windows Defender ECS mappings and make data stream GA
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11249
 - version: "2.1.0"
   changes:
     - description: Tighten IPv4 extraction from IPv4-mapped IPv6 addresses.