From baa39371adca630849be68b78dad01d805a25ed6 Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Thu, 14 Nov 2024 18:11:58 +0100 Subject: [PATCH] [azure] rename azure.platformlogs.properties as .raw when it contains a string value (#11732) Mitigates https://github.com/elastic/integrations/issues/11729 If the `azure.platformlogs.properties` field contains a string instead of the expected object, the pipeline renames it as `azure.platformlogs.properties.raw` field. This allows: - Elasticsearch to index the log events instead of dropping them (status 400) - Users to customize parsing of the `.raw` field in the `logs-azure.platformlogs@custom pipeline` This avoids dropping log events while we build a complete and more robust [invalid JSON](https://github.com/elastic/integrations/issues/11729) handling solution with https://github.com/elastic/integrations/pull/11728 --- packages/azure/changelog.yml | 5 +++ .../test-platformlogs-invalid-json-raw.log | 1 + ...ormlogs-invalid-json-raw.log-expected.json | 44 +++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 6 +++ packages/azure/manifest.yml | 2 +- 5 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log create mode 100644 packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index d6f7e84b529..96c74504218 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,3 +1,8 @@ +- version: "1.19.2" + changes: + - description: Rename the `properties` field to `properties.raw` to avoid parse errors when the `properties` field contains a string. + type: bugfix + link: https://github.com/elastic/integrations/pull/11732 - version: "1.19.1" changes: - description: Fix an error and clarify the docs about the Storage Account container. diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log new file mode 100644 index 00000000000..9defb84b7df --- /dev/null +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log @@ -0,0 +1 @@ +{"time": "2023-03-07T22:19:49Z","resourceId": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2","category": "FunctionAppLogs","operationName": "Microsoft.Web/sites/functions/log","level": "Informational","location": "East US","properties": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}"} \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json new file mode 100644 index 00000000000..78cbe78eb33 --- /dev/null +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-json-raw.log-expected.json @@ -0,0 +1,44 @@ +{ + "expected": [ + { + "@timestamp": "2023-03-07T22:19:49.000Z", + "azure": { + "platformlogs": { + "category": "FunctionAppLogs", + "event_category": "Administrative", + "operation_name": "Microsoft.Web/sites/functions/log", + "properties": { + "raw": "{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}" + } + }, + "resource": { + "group": "MBRANCA-MALFORMED-JSON-RG", + "id": "/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2", + "name": "MBRANCA-HELLO-WORLD2", + "provider": "MICROSOFT.WEB/SITES" + }, + "subscription_id": "0E073EC1-C22F-4488-ADDE-DA35ED609CCD" + }, + "cloud": { + "provider": "azure" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "Microsoft.Web/sites/functions/log", + "kind": "event", + "original": "{\"time\": \"2023-03-07T22:19:49Z\",\"resourceId\": \"/SUBSCRIPTIONS/0E073EC1-C22F-4488-ADDE-DA35ED609CCD/RESOURCEGROUPS/MBRANCA-MALFORMED-JSON-RG/PROVIDERS/MICROSOFT.WEB/SITES/MBRANCA-HELLO-WORLD2\",\"category\": \"FunctionAppLogs\",\"operationName\": \"Microsoft.Web/sites/functions/log\",\"level\": \"Informational\",\"location\": \"East US\",\"properties\": \"{'appName':'mbranca-hello-world2','roleInstance':'A6CE8668-638138213605792171','message':'Executing Functions.hello (Reason=This function was programmatically called via the host APIs., Id=0738eec6-ad5e-48f9-a949-5ac36ba84161)','category':'Function.hello','hostVersion':'4.15.1.1','functionInvocationId':'0738eec6-ad5e-48f9-a949-5ac36ba84161','functionName':'Functions.hello','hostInstanceId':'9eb66127-a244-467e-b6a2-01879ad19da2','level':'Information','levelId': 2,'processId': 55,'eventId': 1,'eventName':'FunctionStarted'}\"}" + }, + "geo": { + "name": "East US" + }, + "log": { + "level": "Informational" + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 24e7b591178..890f8756975 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -54,6 +54,12 @@ processors: field: tags value: ["preserve_original_event"] ignore_failure: true + - rename: + field: azure.platformlogs.properties + if: "ctx.azure?.platformlogs?.properties instanceof String" + target_field: azure.platformlogs.properties.raw + ignore_missing: true + description: 'Rename the field to `properties.raw` to avoid parse errors with the `properties` containing a string.' - rename: field: azure.platformlogs.identity if: "ctx.azure?.platformlogs?.identity instanceof String" diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index 77b66545293..7b0af7d3321 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure Logs -version: 1.19.1 +version: 1.19.2 description: This Elastic integration collects logs from Azure type: integration icons: