[azure] rename azure.platformlogs.properties as .raw when it contains…

… a string value (#11732)

Mitigates #11729

If the `azure.platformlogs.properties` field contains a string instead of the expected object, the pipeline renames it as `azure.platformlogs.properties.raw` field. 

This allows:

- Elasticsearch to index the log events instead of dropping them (status 400)
- Users to customize parsing of the `.raw` field in the `logs-azure.platformlogs@custom pipeline`

This avoids dropping log events while we build a complete and more robust [invalid JSON](#11729) handling solution with #11728

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.19.2"
+  changes:
+    - description: Rename the `properties` field to `properties.raw` to avoid parse errors when the `properties` field contains a string.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11732
 - version: "1.19.1"
   changes:
     - description: Fix an error and clarify the docs about the Storage Account container.

packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log

@@ -0,0 +1 @@
+{"time": "2023-03-07T22:19:49Z","resourceId": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2","category": "FunctionAppLogs","operationName": "Microsoft.Web/sites/functions/log","level": "Informational","location": "East US","properties": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}"}

packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json

@@ -0,0 +1,44 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2023-03-07T22:19:49.000Z",
+            "azure": {
+                "platformlogs": {
+                    "category": "FunctionAppLogs",
+                    "event_category": "Administrative",
+                    "operation_name": "Microsoft.Web/sites/functions/log",
+                    "properties": {
+                        "raw": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}"
+                    }
+                },
+                "resource": {
+                    "group": "MBRANCA-MALFORMED-JSON-RG",
+                    "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2",
+                    "name": "MBRANCA-HELLO-WORLD2",
+                    "provider": "MICROSOFT.WEB/SITES"
+                },
+                "subscription_id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD"
+            },
+            "cloud": {
+                "provider": "azure"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "Microsoft.Web/sites/functions/log",
+                "kind": "event",
+                "original": "{\"time\": \"2023-03-07T22:19:49Z\",\"resourceId\": \"/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2\",\"category\": \"FunctionAppLogs\",\"operationName\": \"Microsoft.Web/sites/functions/log\",\"level\": \"Informational\",\"location\": \"East US\",\"properties\": \"{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}\"}"
+            },
+            "geo": {
+                "name": "East US"
+            },
+            "log": {
+                "level": "Informational"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml

@@ -54,6 +54,12 @@ processors:
             field: tags
             value: ["preserve_original_event"]
             ignore_failure: true
+  - rename:
+      field: azure.platformlogs.properties
+      if: "ctx.azure?.platformlogs?.properties instanceof String"
+      target_field: azure.platformlogs.properties.raw
+      ignore_missing: true
+      description: 'Rename the field to `properties.raw` to avoid parse errors with the `properties` containing a string.'
   - rename:
       field: azure.platformlogs.identity
       if: "ctx.azure?.platformlogs?.identity instanceof String"

packages/azure/manifest.yml

@@ -1,6 +1,6 @@
 name: azure
 title: Azure Logs
-version: 1.19.1
+version: 1.19.2
 description: This Elastic integration collects logs from Azure
 type: integration
 icons: