From 49669c50e31b6d246c64ae37abdfede0ad255076 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Tue, 31 Dec 2024 16:41:23 +0530 Subject: [PATCH 1/3] added support for vnet flow logs --- BlockBlobReader/src/consumer.js | 76 +++++++++++++++++-- .../consumer_build/BlobTaskConsumer/index.js | 76 +++++++++++++++++-- .../DLQTaskConsumer/index.js | 76 +++++++++++++++++-- 3 files changed, 204 insertions(+), 24 deletions(-) diff --git a/BlockBlobReader/src/consumer.js b/BlockBlobReader/src/consumer.js index d5480d52..22d815b4 100644 --- a/BlockBlobReader/src/consumer.js +++ b/BlockBlobReader/src/consumer.js @@ -218,7 +218,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { async function nsgLogsHandler(context, msg, serviceBusTask) { - var jsonArray = []; + let jsonArray = []; msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas try { @@ -236,7 +236,7 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { } - var eventsArr = []; + let eventsArr = []; jsonArray.forEach(function (record) { let version = record.properties.Version; record.properties.flows.forEach(function (rule) { @@ -282,6 +282,66 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { return eventsArr; } +async function networkFlowLogsHandler(context, msg, serviceBusTask) { + + let jsonArray = []; + msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas + + try { + jsonArray = JSON.parse("[" + msg + "]"); + } catch(err) { + let response = getParseableJsonArray(msg, context, serviceBusTask); + jsonArray = response[0]; + let is_success = response[2]; + let newOffset = response[1] + serviceBusTask.startByte; + if (is_success) { + await setAppendBlobOffset(context, serviceBusTask, newOffset); + } else { + return jsonArray; + } + + } + // Format: https://learn.microsoft.com/en-us/azure/network-watcher/vnet-flow-logs-overview#log-format + let eventsArr = []; + jsonArray.forEach(function (record) { + record.flowRecords.flows.forEach(function (acl) { + acl.flowGroups.forEach(function (rule) { + rule.flowTuples.forEach(function (tuple) { + let col = tuple.split(","); + let event = { + time: col[0], // Time stamp of when the flow occurred, in UNIX epoch format. + flowLogGUID: record.flowLogGUID, + category: record.category, + flow_log_resource_id: record.flowLogResourceID, + target_resource_id: record.targetResourceID, + event_name: record.operationName, + acl_id: acl.aclID, + rule_name: rule.rule, + mac: record.macAddress, + src_ip: col[1], + dest_IP: col[2], + src_port: col[3], + dest_port: col[4], + protocol: col[5], + flow_direction: col[6], + flow_state: (col[7] === "" || col[7] === undefined) ? null : col[7], + flow_encryption_status: (col[8] === "" || col[8] === undefined) ? null : col[8], + num_packets_sent_src_to_dest: (col[9] === "" || col[9] === undefined) ? null : col[9], + bytes_sent_src_to_dest: (col[10] === "" || col[10] === undefined) ? null : col[10], + num_packets_sent_dest_to_src: (col[11] === "" || col[11] === undefined) ? null : col[11], + bytes_sent_dest_to_src: (col[12] === "" || col[12] === undefined) ? null : col[12], + version: record.flowLogVersion + + } + eventsArr.push(event); + }); + }); + }); + }); + return eventsArr; +} + + function jsonHandler(context,msg) { // it's assumed that json is well formed {},{} var jsonArray = []; @@ -347,13 +407,13 @@ function messageHandler(serviceBusTask, context, sumoClient) { if (file_ext == serviceBusTask.blobName) { file_ext = "log"; } - var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler}; + var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler, "vnetflowlogs": networkFlowLogsHandler}; if (!(file_ext in msghandler)) { context.log.error("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName); context.done(); return; } - if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent")) { + if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" || serviceBusTask.containerName === "insights-logs-flowlogflowevent")) { // because in json first block and last block remain as it is and azure service adds new block in 2nd last pos if ((serviceBusTask.endByte < JSON_BLOB_HEAD_BYTES + JSON_BLOB_TAIL_BYTES) || (serviceBusTask.endByte == serviceBusTask.startByte)) { context.done(); //rejecting first commit when no data is there data will always be atleast HEAD_BYTES+DATA_BYTES+TAIL_BYTES @@ -365,7 +425,7 @@ function messageHandler(serviceBusTask, context, sumoClient) { } else { serviceBusTask.startByte -= 1; //to remove comma before json object } - file_ext = "nsg"; + file_ext = serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" ? "nsg" : "vnetflowlogs"; } getBlockBlobService(context, serviceBusTask).then(function (blobService) { return getData(serviceBusTask, blobService, context).then(async function (msg) { @@ -385,10 +445,10 @@ function messageHandler(serviceBusTask, context, sumoClient) { context.done(err); }); } else { - if (file_ext == "nsg") { - messageArray = await nsgLogsHandler(context, msg, serviceBusTask); + if (file_ext === "nsg" || file_ext === "vnetflowlogs") { + messageArray = await msghandler[file_ext](context, msg, serviceBusTask); } else { - messageArray = msghandler[file_ext](context,msg); + messageArray = msghandler[file_ext](context, msg); } messageArray.forEach(function (msg) { sumoClient.addData(msg); diff --git a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js index d5480d52..22d815b4 100644 --- a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js +++ b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js @@ -218,7 +218,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { async function nsgLogsHandler(context, msg, serviceBusTask) { - var jsonArray = []; + let jsonArray = []; msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas try { @@ -236,7 +236,7 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { } - var eventsArr = []; + let eventsArr = []; jsonArray.forEach(function (record) { let version = record.properties.Version; record.properties.flows.forEach(function (rule) { @@ -282,6 +282,66 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { return eventsArr; } +async function networkFlowLogsHandler(context, msg, serviceBusTask) { + + let jsonArray = []; + msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas + + try { + jsonArray = JSON.parse("[" + msg + "]"); + } catch(err) { + let response = getParseableJsonArray(msg, context, serviceBusTask); + jsonArray = response[0]; + let is_success = response[2]; + let newOffset = response[1] + serviceBusTask.startByte; + if (is_success) { + await setAppendBlobOffset(context, serviceBusTask, newOffset); + } else { + return jsonArray; + } + + } + // Format: https://learn.microsoft.com/en-us/azure/network-watcher/vnet-flow-logs-overview#log-format + let eventsArr = []; + jsonArray.forEach(function (record) { + record.flowRecords.flows.forEach(function (acl) { + acl.flowGroups.forEach(function (rule) { + rule.flowTuples.forEach(function (tuple) { + let col = tuple.split(","); + let event = { + time: col[0], // Time stamp of when the flow occurred, in UNIX epoch format. + flowLogGUID: record.flowLogGUID, + category: record.category, + flow_log_resource_id: record.flowLogResourceID, + target_resource_id: record.targetResourceID, + event_name: record.operationName, + acl_id: acl.aclID, + rule_name: rule.rule, + mac: record.macAddress, + src_ip: col[1], + dest_IP: col[2], + src_port: col[3], + dest_port: col[4], + protocol: col[5], + flow_direction: col[6], + flow_state: (col[7] === "" || col[7] === undefined) ? null : col[7], + flow_encryption_status: (col[8] === "" || col[8] === undefined) ? null : col[8], + num_packets_sent_src_to_dest: (col[9] === "" || col[9] === undefined) ? null : col[9], + bytes_sent_src_to_dest: (col[10] === "" || col[10] === undefined) ? null : col[10], + num_packets_sent_dest_to_src: (col[11] === "" || col[11] === undefined) ? null : col[11], + bytes_sent_dest_to_src: (col[12] === "" || col[12] === undefined) ? null : col[12], + version: record.flowLogVersion + + } + eventsArr.push(event); + }); + }); + }); + }); + return eventsArr; +} + + function jsonHandler(context,msg) { // it's assumed that json is well formed {},{} var jsonArray = []; @@ -347,13 +407,13 @@ function messageHandler(serviceBusTask, context, sumoClient) { if (file_ext == serviceBusTask.blobName) { file_ext = "log"; } - var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler}; + var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler, "vnetflowlogs": networkFlowLogsHandler}; if (!(file_ext in msghandler)) { context.log.error("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName); context.done(); return; } - if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent")) { + if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" || serviceBusTask.containerName === "insights-logs-flowlogflowevent")) { // because in json first block and last block remain as it is and azure service adds new block in 2nd last pos if ((serviceBusTask.endByte < JSON_BLOB_HEAD_BYTES + JSON_BLOB_TAIL_BYTES) || (serviceBusTask.endByte == serviceBusTask.startByte)) { context.done(); //rejecting first commit when no data is there data will always be atleast HEAD_BYTES+DATA_BYTES+TAIL_BYTES @@ -365,7 +425,7 @@ function messageHandler(serviceBusTask, context, sumoClient) { } else { serviceBusTask.startByte -= 1; //to remove comma before json object } - file_ext = "nsg"; + file_ext = serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" ? "nsg" : "vnetflowlogs"; } getBlockBlobService(context, serviceBusTask).then(function (blobService) { return getData(serviceBusTask, blobService, context).then(async function (msg) { @@ -385,10 +445,10 @@ function messageHandler(serviceBusTask, context, sumoClient) { context.done(err); }); } else { - if (file_ext == "nsg") { - messageArray = await nsgLogsHandler(context, msg, serviceBusTask); + if (file_ext === "nsg" || file_ext === "vnetflowlogs") { + messageArray = await msghandler[file_ext](context, msg, serviceBusTask); } else { - messageArray = msghandler[file_ext](context,msg); + messageArray = msghandler[file_ext](context, msg); } messageArray.forEach(function (msg) { sumoClient.addData(msg); diff --git a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js index d5480d52..22d815b4 100644 --- a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js +++ b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/index.js @@ -218,7 +218,7 @@ async function setAppendBlobOffset(context, serviceBusTask, newOffset) { async function nsgLogsHandler(context, msg, serviceBusTask) { - var jsonArray = []; + let jsonArray = []; msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas try { @@ -236,7 +236,7 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { } - var eventsArr = []; + let eventsArr = []; jsonArray.forEach(function (record) { let version = record.properties.Version; record.properties.flows.forEach(function (rule) { @@ -282,6 +282,66 @@ async function nsgLogsHandler(context, msg, serviceBusTask) { return eventsArr; } +async function networkFlowLogsHandler(context, msg, serviceBusTask) { + + let jsonArray = []; + msg = msg.trim().replace(/(^,)|(,$)/g, ""); //removing trailing spaces,newlines and leftover commas + + try { + jsonArray = JSON.parse("[" + msg + "]"); + } catch(err) { + let response = getParseableJsonArray(msg, context, serviceBusTask); + jsonArray = response[0]; + let is_success = response[2]; + let newOffset = response[1] + serviceBusTask.startByte; + if (is_success) { + await setAppendBlobOffset(context, serviceBusTask, newOffset); + } else { + return jsonArray; + } + + } + // Format: https://learn.microsoft.com/en-us/azure/network-watcher/vnet-flow-logs-overview#log-format + let eventsArr = []; + jsonArray.forEach(function (record) { + record.flowRecords.flows.forEach(function (acl) { + acl.flowGroups.forEach(function (rule) { + rule.flowTuples.forEach(function (tuple) { + let col = tuple.split(","); + let event = { + time: col[0], // Time stamp of when the flow occurred, in UNIX epoch format. + flowLogGUID: record.flowLogGUID, + category: record.category, + flow_log_resource_id: record.flowLogResourceID, + target_resource_id: record.targetResourceID, + event_name: record.operationName, + acl_id: acl.aclID, + rule_name: rule.rule, + mac: record.macAddress, + src_ip: col[1], + dest_IP: col[2], + src_port: col[3], + dest_port: col[4], + protocol: col[5], + flow_direction: col[6], + flow_state: (col[7] === "" || col[7] === undefined) ? null : col[7], + flow_encryption_status: (col[8] === "" || col[8] === undefined) ? null : col[8], + num_packets_sent_src_to_dest: (col[9] === "" || col[9] === undefined) ? null : col[9], + bytes_sent_src_to_dest: (col[10] === "" || col[10] === undefined) ? null : col[10], + num_packets_sent_dest_to_src: (col[11] === "" || col[11] === undefined) ? null : col[11], + bytes_sent_dest_to_src: (col[12] === "" || col[12] === undefined) ? null : col[12], + version: record.flowLogVersion + + } + eventsArr.push(event); + }); + }); + }); + }); + return eventsArr; +} + + function jsonHandler(context,msg) { // it's assumed that json is well formed {},{} var jsonArray = []; @@ -347,13 +407,13 @@ function messageHandler(serviceBusTask, context, sumoClient) { if (file_ext == serviceBusTask.blobName) { file_ext = "log"; } - var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler}; + var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler, "vnetflowlogs": networkFlowLogsHandler}; if (!(file_ext in msghandler)) { context.log.error("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName); context.done(); return; } - if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent")) { + if ((file_ext === "json") && (serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" || serviceBusTask.containerName === "insights-logs-flowlogflowevent")) { // because in json first block and last block remain as it is and azure service adds new block in 2nd last pos if ((serviceBusTask.endByte < JSON_BLOB_HEAD_BYTES + JSON_BLOB_TAIL_BYTES) || (serviceBusTask.endByte == serviceBusTask.startByte)) { context.done(); //rejecting first commit when no data is there data will always be atleast HEAD_BYTES+DATA_BYTES+TAIL_BYTES @@ -365,7 +425,7 @@ function messageHandler(serviceBusTask, context, sumoClient) { } else { serviceBusTask.startByte -= 1; //to remove comma before json object } - file_ext = "nsg"; + file_ext = serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent" ? "nsg" : "vnetflowlogs"; } getBlockBlobService(context, serviceBusTask).then(function (blobService) { return getData(serviceBusTask, blobService, context).then(async function (msg) { @@ -385,10 +445,10 @@ function messageHandler(serviceBusTask, context, sumoClient) { context.done(err); }); } else { - if (file_ext == "nsg") { - messageArray = await nsgLogsHandler(context, msg, serviceBusTask); + if (file_ext === "nsg" || file_ext === "vnetflowlogs") { + messageArray = await msghandler[file_ext](context, msg, serviceBusTask); } else { - messageArray = msghandler[file_ext](context,msg); + messageArray = msghandler[file_ext](context, msg); } messageArray.forEach(function (msg) { sumoClient.addData(msg); From 2615906a1792c1c116400d941b8debbd386bbae5 Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Wed, 1 Jan 2025 12:29:30 +0530 Subject: [PATCH 2/3] added fixtures and modified tests --- BlockBlobReader/src/CHANGELOG.md | 32 ---------- BlockBlobReader/src/blobreaderzipdeploy.json | 6 +- BlockBlobReader/src/create_zip.sh | 2 +- ...lob_fixtures_networkinterfaceflowlogs.json | 1 + .../tests/blob_fixtures_subnetflowlogs.json | 1 + .../tests/blob_fixtures_vnetflowlogs.json | 1 + BlockBlobReader/tests/test_blobreader.py | 59 ++++++++++++------- 7 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 BlockBlobReader/src/CHANGELOG.md create mode 100644 BlockBlobReader/tests/blob_fixtures_networkinterfaceflowlogs.json create mode 100644 BlockBlobReader/tests/blob_fixtures_subnetflowlogs.json create mode 100644 BlockBlobReader/tests/blob_fixtures_vnetflowlogs.json diff --git a/BlockBlobReader/src/CHANGELOG.md b/BlockBlobReader/src/CHANGELOG.md deleted file mode 100644 index 7e644eab..00000000 --- a/BlockBlobReader/src/CHANGELOG.md +++ /dev/null @@ -1,32 +0,0 @@ -# CHANGELOG - -All notable changes to this project will be documented in this file. -This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/). - - - -## Unreleased ---- - -### New - -### Changes - -### Fixes - -### Breaks - - -## 2.0.0 - (2023-05-16) ---- - -### New -* Updated BlockBlob Reader with Azure Function Runtime 4.x - -## 1.0.0 - (2018-05-25) ---- - -### New -* Initial release of BlockBlob Reader with Azure Function Runtime 1.x - - diff --git a/BlockBlobReader/src/blobreaderzipdeploy.json b/BlockBlobReader/src/blobreaderzipdeploy.json index 2e11d64c..b017b6d2 100644 --- a/BlockBlobReader/src/blobreaderzipdeploy.json +++ b/BlockBlobReader/src/blobreaderzipdeploy.json @@ -419,7 +419,7 @@ "[variables('BlobReader_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskproducer4.1.4.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskproducer4.1.5.zip", "appOffline": true } } @@ -512,7 +512,7 @@ "[variables('blobreaderconsumer_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskconsumer4.1.4.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/taskconsumer4.1.5.zip", "appOffline": true } } @@ -609,7 +609,7 @@ "[variables('DLQProcessor_resourceId')]" ], "properties": { - "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/dlqprocessor4.1.4.zip", + "packageUri": "https://appdev-cloudformation-templates.s3.amazonaws.com/AzureBlobReader/dlqprocessor4.1.5.zip", "appOffline": true } } diff --git a/BlockBlobReader/src/create_zip.sh b/BlockBlobReader/src/create_zip.sh index f85c9f7e..289775ab 100755 --- a/BlockBlobReader/src/create_zip.sh +++ b/BlockBlobReader/src/create_zip.sh @@ -80,7 +80,7 @@ else fi echo "creating zip" -version="4.1.4" +version="4.1.5" producer_zip_file="taskproducer$version.zip" consumer_zip_file="taskconsumer$version.zip" dlqprocessor_zip_file="dlqprocessor$version.zip" diff --git a/BlockBlobReader/tests/blob_fixtures_networkinterfaceflowlogs.json b/BlockBlobReader/tests/blob_fixtures_networkinterfaceflowlogs.json new file mode 100644 index 00000000..69bdd999 --- /dev/null +++ b/BlockBlobReader/tests/blob_fixtures_networkinterfaceflowlogs.json @@ -0,0 +1 @@ +{"records":[{"time":"2024-12-31T09:13:37.7570617Z","flowLogGUID":"1e98aac7-0e3a-4365-8d28-a15797fcb58b","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1519-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkInterfaces/labsvm1519","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636357995,10.1.0.4,20.10.127.193,56354,443,6,O,E,NX,8,5183,9,6707","1735636357995,10.1.0.4,20.10.127.193,56354,443,6,O,B,NX,0,0,0,0","1735636363510,10.1.0.4,20.10.127.193,56354,443,6,O,E,NX,10,5291,10,6809","1735636371724,10.1.0.4,20.242.181.1,56346,443,6,O,E,NX,9,2313,15,13794","1735636371724,10.1.0.4,20.242.181.1,56346,443,6,O,B,NX,0,0,0,0","1735636377136,10.1.0.4,20.242.181.1,56346,443,6,O,E,NX,12,2506,17,13998","1735636404728,10.1.0.4,40.119.6.228,123,123,17,O,E,NX,27347,2461230,27343,3609276","1735636404728,10.1.0.4,40.119.6.228,123,123,17,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636357306,35.203.210.97,10.1.0.4,53909,9974,6,I,D,NX,0,0,0,0","1735636364466,162.216.149.203,10.1.0.4,53851,15060,17,I,D,NX,0,0,0,0","1735636364692,35.203.211.115,10.1.0.4,51770,2587,6,I,D,NX,0,0,0,0","1735636364692,35.203.211.42,10.1.0.4,54206,46866,6,I,D,NX,0,0,0,0","1735636370861,162.216.149.19,10.1.0.4,53990,45721,6,I,D,NX,0,0,0,0","1735636379799,154.213.184.66,10.1.0.4,57664,39881,6,I,D,NX,0,0,0,0","1735636383138,162.216.150.165,10.1.0.4,51348,49158,6,I,D,NX,0,0,0,0","1735636385449,80.82.68.110,10.1.0.4,53673,999,6,I,D,NX,0,0,0,0","1735636390706,207.90.244.11,10.1.0.4,29011,9090,6,I,D,NX,0,0,0,0","1735636390929,65.49.1.15,10.1.0.4,59338,8888,6,I,D,NX,0,0,0,0","1735636394103,205.210.31.27,10.1.0.4,52853,4433,6,I,D,NX,0,0,0,0","1735636396181,162.216.149.65,10.1.0.4,49429,31288,6,I,D,NX,0,0,0,0","1735636400862,91.191.209.74,10.1.0.4,46507,6446,6,I,D,NX,0,0,0,0","1735636409116,35.203.211.132,10.1.0.4,53451,21240,6,I,D,NX,0,0,0,0","1735636412366,94.232.47.83,10.1.0.4,50130,4921,6,I,D,NX,0,0,0,0","1735636412833,66.228.40.131,10.1.0.4,61000,995,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:14:37.7656867Z","flowLogGUID":"1e98aac7-0e3a-4365-8d28-a15797fcb58b","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1519-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkInterfaces/labsvm1519","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"rule":"PlatformRule","flowTuples":["1735636429638,40.87.168.8,10.1.0.4,23456,29554,6,I,E,NX,1,128,0,0","1735636454117,52.136.4.16,10.1.0.4,23456,23551,6,I,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636453889,10.1.0.4,13.69.239.78,56429,443,6,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636419276,147.185.132.230,10.1.0.4,49397,34437,6,I,D,NX,0,0,0,0","1735636423679,198.235.24.136,10.1.0.4,50813,5289,6,I,D,NX,0,0,0,0","1735636430993,172.214.115.102,10.1.0.4,47244,8443,6,I,D,NX,0,0,0,0","1735636432699,172.104.219.195,10.1.0.4,61000,465,6,I,D,NX,0,0,0,0","1735636436888,79.124.49.130,10.1.0.4,48987,5927,6,I,D,NX,0,0,0,0","1735636454377,35.203.210.228,10.1.0.4,52520,48566,6,I,D,NX,0,0,0,0","1735636459857,35.203.211.175,10.1.0.4,53174,9336,6,I,D,NX,0,0,0,0","1735636460833,205.210.31.250,10.1.0.4,55854,2085,6,I,D,NX,0,0,0,0","1735636469698,198.235.24.144,10.1.0.4,54729,9444,6,I,D,NX,0,0,0,0","1735636470560,162.216.150.229,10.1.0.4,55850,49499,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:15:37.7768685Z","flowLogGUID":"1e98aac7-0e3a-4365-8d28-a15797fcb58b","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1519-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkInterfaces/labsvm1519","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636505641,10.1.0.4,40.83.240.146,49715,443,6,O,E,NX,1,97,1,108","1735636505641,10.1.0.4,40.83.240.146,49715,443,6,O,B,NX,0,0,0,0","1735636513957,10.1.0.4,172.202.64.10,56457,443,6,O,B,NX,0,0,0,0","1735636519430,10.1.0.4,172.202.64.10,56457,443,6,O,E,NX,16,4575,27,27252","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,B,NX,0,0,0,0","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,E,NX,2,120,1,108","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636491732,35.203.210.201,10.1.0.4,53528,24015,6,I,D,NX,0,0,0,0","1735636505601,74.82.47.36,10.1.0.4,34507,2095,6,I,D,NX,0,0,0,0","1735636510680,147.185.133.64,10.1.0.4,56445,9778,6,I,D,NX,0,0,0,0","1735636512138,185.113.223.82,10.1.0.4,34274,443,6,I,D,NX,0,0,0,0","1735636521303,162.216.150.179,10.1.0.4,51822,9612,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:15:37.7768685Z","flowLogGUID":"1e98aac7-0e3a-4365-8d28-a15797fcb58b","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1519-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkInterfaces/labsvm1519","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636505641,10.1.0.4,40.83.240.146,49715,443,6,O,E,NX,1,97,1,108","1735636505641,10.1.0.4,40.83.240.146,49715,443,6,O,B,NX,0,0,0,0","1735636513957,10.1.0.4,172.202.64.10,56457,443,6,O,B,NX,0,0,0,0","1735636519430,10.1.0.4,172.202.64.10,56457,443,6,O,E,NX,16,4575,27,27252","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,B,NX,0,0,0,0","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,E,NX,2,120,1,108","1735636533303,10.1.0.4,20.189.173.10,56466,443,6,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636491732,35.203.210.201,10.1.0.4,53528,24015,6,I,D,NX,0,0,0,0","1735636505601,74.82.47.36,10.1.0.4,34507,2095,6,I,D,NX,0,0,0,0","1735636510680,147.185.133.64,10.1.0.4,56445,9778,6,I,D,NX,0,0,0,0","1735636512138,185.113.223.82,10.1.0.4,34274,443,6,I,D,NX,0,0,0,0","1735636521303,162.216.150.179,10.1.0.4,51822,9612,6,I,D,NX,0,0,0,0"]}]}]}}]} \ No newline at end of file diff --git a/BlockBlobReader/tests/blob_fixtures_subnetflowlogs.json b/BlockBlobReader/tests/blob_fixtures_subnetflowlogs.json new file mode 100644 index 00000000..e521a477 --- /dev/null +++ b/BlockBlobReader/tests/blob_fixtures_subnetflowlogs.json @@ -0,0 +1 @@ +{"records":[{"time":"2024-12-31T09:11:37.7568150Z","flowLogGUID":"36094b6b-6999-48cf-8052-f8815f6732d9","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/DEFAULT-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet/subnets/default","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"rule":"PlatformRule","flowTuples":["1735636278979,40.87.168.8,10.1.0.4,23456,29544,6,I,E,NX,1,128,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636251391,10.1.0.4,20.242.181.1,56346,443,6,O,B,NX,0,0,0,0","1735636253344,10.1.0.4,104.208.16.91,56347,443,6,O,B,NX,0,0,0,0","1735636259204,10.1.0.4,104.208.16.91,56347,443,6,O,E,NX,12,5045,12,5902","1735636268734,10.1.0.4,20.10.127.193,56354,443,6,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636241259,45.79.114.248,10.1.0.4,50971,16001,6,I,D,NX,0,0,0,0","1735636251726,154.213.184.18,10.1.0.4,40878,8888,6,I,D,NX,0,0,0,0","1735636259001,162.216.149.21,10.1.0.4,55950,12128,6,I,D,NX,0,0,0,0","1735636264523,206.168.35.62,10.1.0.4,23268,990,6,I,D,NX,0,0,0,0","1735636271942,162.216.150.57,10.1.0.4,49731,9793,6,I,D,NX,0,0,0,0","1735636272594,143.42.1.128,10.1.0.4,46816,853,6,I,D,NX,0,0,0,0","1735636277859,162.216.150.149,10.1.0.4,56839,50069,6,I,D,NX,0,0,0,0","1735636278685,167.94.145.19,10.1.0.4,51253,4433,6,I,D,NX,0,0,0,0","1735636285012,162.216.150.42,10.1.0.4,53038,11000,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:12:37.7569719Z","flowLogGUID":"36094b6b-6999-48cf-8052-f8815f6732d9","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/DEFAULT-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet/subnets/default","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"rule":"PlatformRule","flowTuples":["1735636300432,40.87.164.0,10.1.0.4,23456,27583,6,I,E,NX,1,128,0,0","1735636330574,40.87.164.0,10.1.0.4,23456,27585,6,I,E,NX,1,128,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,B,NX,0,0,0,0","1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,E,NX,2,120,1,108","1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,B,NX,0,0,0,0","1735636303451,10.1.0.4,20.189.173.13,56368,443,6,O,E,NX,12,10153,14,6076","1735636340707,10.1.0.4,40.119.6.228,123,123,17,O,E,NX,2,180,2,264","1735636340707,10.1.0.4,40.119.6.228,123,123,17,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636295623,47.254.243.146,10.1.0.4,29344,18080,6,I,D,NX,0,0,0,0","1735636296353,167.94.138.107,10.1.0.4,22657,59195,6,I,D,NX,0,0,0,0","1735636299336,45.9.75.232,10.1.0.4,54794,23,6,I,D,NX,0,0,0,0","1735636306019,162.216.150.199,10.1.0.4,55173,9382,6,I,D,NX,0,0,0,0","1735636306994,35.203.210.52,10.1.0.4,56010,33168,6,I,D,NX,0,0,0,0","1735636317366,206.168.35.115,10.1.0.4,63905,179,6,I,D,NX,0,0,0,0","1735636341803,137.184.238.110,10.1.0.4,50970,22,6,I,D,NX,0,0,0,0","1735636343013,162.216.150.252,10.1.0.4,53462,49577,6,I,D,NX,0,0,0,0","1735636344656,193.41.206.156,10.1.0.4,44848,8728,6,I,D,NX,0,0,0,0","1735636348801,118.26.105.116,10.1.0.4,40504,1433,6,I,D,NX,0,0,0,0","1735636350223,35.203.210.89,10.1.0.4,56051,13651,6,I,D,NX,0,0,0,0","1735636351469,147.185.133.108,10.1.0.4,55432,2232,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:11:37.7568150Z","flowLogGUID":"36094b6b-6999-48cf-8052-f8815f6732d9","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/DEFAULT-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet/subnets/default","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"rule":"PlatformRule","flowTuples":["1735636278979,40.87.168.8,10.1.0.4,23456,29544,6,I,E,NX,1,128,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636251391,10.1.0.4,20.242.181.1,56346,443,6,O,B,NX,0,0,0,0","1735636253344,10.1.0.4,104.208.16.91,56347,443,6,O,B,NX,0,0,0,0","1735636259204,10.1.0.4,104.208.16.91,56347,443,6,O,E,NX,12,5045,12,5902","1735636268734,10.1.0.4,20.10.127.193,56354,443,6,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636241259,45.79.114.248,10.1.0.4,50971,16001,6,I,D,NX,0,0,0,0","1735636251726,154.213.184.18,10.1.0.4,40878,8888,6,I,D,NX,0,0,0,0","1735636259001,162.216.149.21,10.1.0.4,55950,12128,6,I,D,NX,0,0,0,0","1735636264523,206.168.35.62,10.1.0.4,23268,990,6,I,D,NX,0,0,0,0","1735636271942,162.216.150.57,10.1.0.4,49731,9793,6,I,D,NX,0,0,0,0","1735636272594,143.42.1.128,10.1.0.4,46816,853,6,I,D,NX,0,0,0,0","1735636277859,162.216.150.149,10.1.0.4,56839,50069,6,I,D,NX,0,0,0,0","1735636278685,167.94.145.19,10.1.0.4,51253,4433,6,I,D,NX,0,0,0,0","1735636285012,162.216.150.42,10.1.0.4,53038,11000,6,I,D,NX,0,0,0,0"]}]}]}},{"time":"2024-12-31T09:12:37.7569719Z","flowLogGUID":"36094b6b-6999-48cf-8052-f8815f6732d9","macAddress":"000D3A359573","category":"FlowLogFlowEvent","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/DEFAULT-AZURE-LABS-001-FLOWLOG","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet/subnets/default","flowLogVersion":4,"operationName":"FlowLogFlowEvent","flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"rule":"PlatformRule","flowTuples":["1735636300432,40.87.164.0,10.1.0.4,23456,27583,6,I,E,NX,1,128,0,0","1735636330574,40.87.164.0,10.1.0.4,23456,27585,6,I,E,NX,1,128,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_AllowInternetOutBound","flowTuples":["1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,B,NX,0,0,0,0","1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,E,NX,2,120,1,108","1735636297461,10.1.0.4,20.189.173.13,56368,443,6,O,B,NX,0,0,0,0","1735636303451,10.1.0.4,20.189.173.13,56368,443,6,O,E,NX,12,10153,14,6076","1735636340707,10.1.0.4,40.119.6.228,123,123,17,O,E,NX,2,180,2,264","1735636340707,10.1.0.4,40.119.6.228,123,123,17,O,B,NX,0,0,0,0"]}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"rule":"DefaultRule_DenyAllInBound","flowTuples":["1735636295623,47.254.243.146,10.1.0.4,29344,18080,6,I,D,NX,0,0,0,0","1735636296353,167.94.138.107,10.1.0.4,22657,59195,6,I,D,NX,0,0,0,0","1735636299336,45.9.75.232,10.1.0.4,54794,23,6,I,D,NX,0,0,0,0","1735636306019,162.216.150.199,10.1.0.4,55173,9382,6,I,D,NX,0,0,0,0","1735636306994,35.203.210.52,10.1.0.4,56010,33168,6,I,D,NX,0,0,0,0","1735636317366,206.168.35.115,10.1.0.4,63905,179,6,I,D,NX,0,0,0,0","1735636341803,137.184.238.110,10.1.0.4,50970,22,6,I,D,NX,0,0,0,0","1735636343013,162.216.150.252,10.1.0.4,53462,49577,6,I,D,NX,0,0,0,0","1735636344656,193.41.206.156,10.1.0.4,44848,8728,6,I,D,NX,0,0,0,0","1735636348801,118.26.105.116,10.1.0.4,40504,1433,6,I,D,NX,0,0,0,0","1735636350223,35.203.210.89,10.1.0.4,56051,13651,6,I,D,NX,0,0,0,0","1735636351469,147.185.133.108,10.1.0.4,55432,2232,6,I,D,NX,0,0,0,0"]}]}]}}]} \ No newline at end of file diff --git a/BlockBlobReader/tests/blob_fixtures_vnetflowlogs.json b/BlockBlobReader/tests/blob_fixtures_vnetflowlogs.json new file mode 100644 index 00000000..eaaa55e8 --- /dev/null +++ b/BlockBlobReader/tests/blob_fixtures_vnetflowlogs.json @@ -0,0 +1 @@ +{"records":[{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733975962447,10.1.0.4,40.119.6.228,123,123,17,O,C,NX,4,360,4,528","1733975972606,10.1.0.4,20.10.127.193,57892,443,6,O,E,NX,10,5291,11,6911","1733976005817,10.1.0.4,40.83.240.146,49715,443,6,O,C,NX,1,97,1,108"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733975966605,64.62.197.181,10.1.0.4,19795,2123,17,I,D,NX,0,0,0,0","1733975968615,198.235.24.35,10.1.0.4,56058,52869,6,I,D,NX,0,0,0,0","1733975968747,162.216.150.207,10.1.0.4,54328,46229,6,I,D,NX,0,0,0,0","1733975973469,79.110.62.68,10.1.0.4,47461,4242,6,I,D,NX,0,0,0,0","1733975976762,103.102.230.6,10.1.0.4,45451,8728,6,I,D,NX,0,0,0,0","1733975980096,162.216.149.126,10.1.0.4,49791,8055,6,I,D,NX,0,0,0,0","1733975986586,35.203.210.177,10.1.0.4,54105,9904,6,I,D,NX,0,0,0,0","1733975987266,147.185.133.9,10.1.0.4,50241,60241,6,I,D,NX,0,0,0,0","1733975989126,194.180.49.219,10.1.0.4,58856,4709,6,I,D,NX,0,0,0,0","1733975993431,167.94.145.110,10.1.0.4,44454,5002,6,I,D,NX,0,0,0,0","1733975994464,167.94.145.110,10.1.0.4,44472,5002,6,I,D,NX,0,0,0,0","1733975994642,35.203.210.50,10.1.0.4,55806,50080,6,I,D,NX,0,0,0,0","1733975995496,167.94.145.110,10.1.0.4,44472,5002,6,I,D,NX,0,0,0,0","1733975995565,167.94.145.110,10.1.0.4,44498,5002,6,I,D,NX,0,0,0,0","1733975996585,167.94.145.110,10.1.0.4,44498,5002,6,I,D,NX,0,0,0,0","1733976002076,194.180.49.220,10.1.0.4,58824,6664,6,I,D,NX,0,0,0,0","1733976003320,147.185.133.226,10.1.0.4,54062,4917,6,I,D,NX,0,0,0,0","1733976009361,184.105.247.238,10.1.0.4,45641,143,6,I,D,NX,0,0,0,0","1733976014832,147.185.133.12,10.1.0.4,53554,24172,6,I,D,NX,0,0,0,0","1733976015168,103.102.230.5,10.1.0.4,55194,8728,6,I,D,NX,0,0,0,0","1733976020181,194.180.49.71,10.1.0.4,57643,16938,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:00:24.7385180Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976032611,40.87.168.8,10.1.0.4,23456,29382,6,I,E,NX,1,128,0,0","1733976044771,40.87.168.8,10.1.0.4,23456,29400,6,I,B,NX,0,0,0,0","1733976077879,40.87.168.8,10.1.0.4,23456,29385,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976029250,205.210.31.170,10.1.0.4,55336,49502,6,I,D,NX,0,0,0,0","1733976039201,206.168.35.58,10.1.0.4,57258,500,17,I,D,NX,0,0,0,0","1733976043796,194.180.49.71,10.1.0.4,57643,18443,6,I,D,NX,0,0,0,0","1733976046304,52.189.78.2,10.1.0.4,54096,9529,6,I,D,NX,0,0,0,0","1733976055514,185.91.127.43,10.1.0.4,43334,8001,6,I,D,NX,0,0,0,0","1733976057371,35.203.211.13,10.1.0.4,51769,9428,6,I,D,NX,0,0,0,0","1733976058141,89.248.165.110,10.1.0.4,54367,254,6,I,D,NX,0,0,0,0","1733976060079,35.203.211.21,10.1.0.4,50647,3722,6,I,D,NX,0,0,0,0","1733976063763,147.185.133.92,10.1.0.4,53169,3445,6,I,D,NX,0,0,0,0","1733976072769,162.216.150.201,10.1.0.4,56967,8821,6,I,D,NX,0,0,0,0","1733976078869,194.180.49.220,10.1.0.4,58824,8143,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:01:24.7434270Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976090136,40.87.168.8,10.1.0.4,23456,29403,6,I,B,NX,0,0,0,0","1733976122944,40.87.164.0,10.1.0.4,23456,27457,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976095813,10.1.0.4,13.89.179.11,57971,443,6,O,B,NX,0,0,0,0","1733976101509,10.1.0.4,13.89.179.11,57971,443,6,O,E,NX,15,25735,16,6312"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976094709,194.180.49.219,10.1.0.4,58856,1739,6,I,D,NX,0,0,0,0","1733976094770,194.180.49.71,10.1.0.4,57643,11981,6,I,D,NX,0,0,0,0","1733976097546,162.216.150.195,10.1.0.4,56487,8587,6,I,D,NX,0,0,0,0","1733976098154,216.218.206.105,10.1.0.4,41536,6443,6,I,D,NX,0,0,0,0","1733976098509,79.110.62.68,10.1.0.4,47461,5578,6,I,D,NX,0,0,0,0","1733976101721,199.45.154.180,10.1.0.4,19145,1645,17,I,D,NX,0,0,0,0","1733976102809,79.110.62.162,10.1.0.4,45666,41369,6,I,D,NX,0,0,0,0","1733976106524,23.94.41.179,10.1.0.4,7025,5480,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:02:24.7518798Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976142353,52.136.4.16,10.1.0.4,23456,23064,6,I,E,NX,1,128,0,0","1733976153201,40.87.164.0,10.1.0.4,23456,27459,6,I,E,NX,1,128,0,0","1733976195979,40.87.168.8,10.1.0.4,23456,29410,6,I,B,NX,0,0,0,0","1733976198617,40.87.164.0,10.1.0.4,23456,27462,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976155844,10.1.0.4,172.202.65.10,57999,443,6,O,B,NX,0,0,0,0","1733976161288,10.1.0.4,172.202.65.10,57999,443,6,O,E,NX,14,3027,26,27173"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976151178,157.10.52.252,10.1.0.4,53542,3389,6,I,D,NX,0,0,0,0"],"rule":"UserRule_MicrosoftDefenderForCloud-JITRule_2017955537_88ABE006EA9C4130BF9065D4405169C7"},{"flowTuples":["1733976145510,79.110.62.68,10.1.0.4,47461,61523,6,I,D,NX,0,0,0,0","1733976150479,35.203.211.110,10.1.0.4,52556,24430,6,I,D,NX,0,0,0,0","1733976150601,92.255.85.51,10.1.0.4,51564,19623,6,I,D,NX,0,0,0,0","1733976153234,147.185.133.23,10.1.0.4,50577,46266,6,I,D,NX,0,0,0,0","1733976155135,194.180.49.219,10.1.0.4,58856,4650,6,I,D,NX,0,0,0,0","1733976156681,147.185.132.127,10.1.0.4,55295,45598,6,I,D,NX,0,0,0,0","1733976168973,147.185.133.93,10.1.0.4,52167,47512,6,I,D,NX,0,0,0,0","1733976170784,162.216.149.152,10.1.0.4,55770,54266,6,I,D,NX,0,0,0,0","1733976174863,206.168.35.187,10.1.0.4,24306,5061,6,I,D,NX,0,0,0,0","1733976176727,79.110.62.177,10.1.0.4,43008,12001,6,I,D,NX,0,0,0,0","1733976178593,13.64.193.117,10.1.0.4,51250,1337,6,I,D,NX,0,0,0,0","1733976179579,162.142.125.240,10.1.0.4,13609,52641,17,I,D,NX,0,0,0,0","1733976179751,162.216.149.27,10.1.0.4,49786,9556,6,I,D,NX,0,0,0,0","1733976180099,79.110.62.68,10.1.0.4,47461,61720,6,I,D,NX,0,0,0,0","1733976181004,35.203.210.156,10.1.0.4,56941,28444,6,I,D,NX,0,0,0,0","1733976181039,35.203.211.95,10.1.0.4,54031,9735,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976193409,193.68.89.3,10.1.0.4,49550,9055,6,I,D,NX,0,0,0,0","1733976196552,162.216.149.211,10.1.0.4,49834,9261,6,I,D,NX,0,0,0,0","1733976197528,205.210.31.184,10.1.0.4,55197,5909,6,I,D,NX,0,0,0,0","1733976198463,147.185.133.102,10.1.0.4,50085,23917,6,I,D,NX,0,0,0,0","1733976198737,79.110.62.68,10.1.0.4,47461,60741,6,I,D,NX,0,0,0,0","1733976199440,185.180.141.43,10.1.0.4,20730,6667,6,I,D,NX,0,0,0,0","1733976201177,194.180.49.220,10.1.0.4,58824,7092,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:03:24.7776580Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733975962447,10.1.0.4,40.119.6.228,123,123,17,O,C,NX,4,360,4,528","1733975972606,10.1.0.4,20.10.127.193,57892,443,6,O,E,NX,10,5291,11,6911","1733976005817,10.1.0.4,40.83.240.146,49715,443,6,O,C,NX,1,97,1,108"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733975966605,64.62.197.181,10.1.0.4,19795,2123,17,I,D,NX,0,0,0,0","1733975968615,198.235.24.35,10.1.0.4,56058,52869,6,I,D,NX,0,0,0,0","1733975968747,162.216.150.207,10.1.0.4,54328,46229,6,I,D,NX,0,0,0,0","1733975973469,79.110.62.68,10.1.0.4,47461,4242,6,I,D,NX,0,0,0,0","1733975976762,103.102.230.6,10.1.0.4,45451,8728,6,I,D,NX,0,0,0,0","1733975980096,162.216.149.126,10.1.0.4,49791,8055,6,I,D,NX,0,0,0,0","1733975986586,35.203.210.177,10.1.0.4,54105,9904,6,I,D,NX,0,0,0,0","1733975987266,147.185.133.9,10.1.0.4,50241,60241,6,I,D,NX,0,0,0,0","1733975989126,194.180.49.219,10.1.0.4,58856,4709,6,I,D,NX,0,0,0,0","1733975993431,167.94.145.110,10.1.0.4,44454,5002,6,I,D,NX,0,0,0,0","1733975994464,167.94.145.110,10.1.0.4,44472,5002,6,I,D,NX,0,0,0,0","1733975994642,35.203.210.50,10.1.0.4,55806,50080,6,I,D,NX,0,0,0,0","1733975995496,167.94.145.110,10.1.0.4,44472,5002,6,I,D,NX,0,0,0,0","1733975995565,167.94.145.110,10.1.0.4,44498,5002,6,I,D,NX,0,0,0,0","1733975996585,167.94.145.110,10.1.0.4,44498,5002,6,I,D,NX,0,0,0,0","1733976002076,194.180.49.220,10.1.0.4,58824,6664,6,I,D,NX,0,0,0,0","1733976003320,147.185.133.226,10.1.0.4,54062,4917,6,I,D,NX,0,0,0,0","1733976009361,184.105.247.238,10.1.0.4,45641,143,6,I,D,NX,0,0,0,0","1733976014832,147.185.133.12,10.1.0.4,53554,24172,6,I,D,NX,0,0,0,0","1733976015168,103.102.230.5,10.1.0.4,55194,8728,6,I,D,NX,0,0,0,0","1733976020181,194.180.49.71,10.1.0.4,57643,16938,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:00:24.7385180Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976032611,40.87.168.8,10.1.0.4,23456,29382,6,I,E,NX,1,128,0,0","1733976044771,40.87.168.8,10.1.0.4,23456,29400,6,I,B,NX,0,0,0,0","1733976077879,40.87.168.8,10.1.0.4,23456,29385,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976029250,205.210.31.170,10.1.0.4,55336,49502,6,I,D,NX,0,0,0,0","1733976039201,206.168.35.58,10.1.0.4,57258,500,17,I,D,NX,0,0,0,0","1733976043796,194.180.49.71,10.1.0.4,57643,18443,6,I,D,NX,0,0,0,0","1733976046304,52.189.78.2,10.1.0.4,54096,9529,6,I,D,NX,0,0,0,0","1733976055514,185.91.127.43,10.1.0.4,43334,8001,6,I,D,NX,0,0,0,0","1733976057371,35.203.211.13,10.1.0.4,51769,9428,6,I,D,NX,0,0,0,0","1733976058141,89.248.165.110,10.1.0.4,54367,254,6,I,D,NX,0,0,0,0","1733976060079,35.203.211.21,10.1.0.4,50647,3722,6,I,D,NX,0,0,0,0","1733976063763,147.185.133.92,10.1.0.4,53169,3445,6,I,D,NX,0,0,0,0","1733976072769,162.216.150.201,10.1.0.4,56967,8821,6,I,D,NX,0,0,0,0","1733976078869,194.180.49.220,10.1.0.4,58824,8143,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:01:24.7434270Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976090136,40.87.168.8,10.1.0.4,23456,29403,6,I,B,NX,0,0,0,0","1733976122944,40.87.164.0,10.1.0.4,23456,27457,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976095813,10.1.0.4,13.89.179.11,57971,443,6,O,B,NX,0,0,0,0","1733976101509,10.1.0.4,13.89.179.11,57971,443,6,O,E,NX,15,25735,16,6312"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976094709,194.180.49.219,10.1.0.4,58856,1739,6,I,D,NX,0,0,0,0","1733976094770,194.180.49.71,10.1.0.4,57643,11981,6,I,D,NX,0,0,0,0","1733976097546,162.216.150.195,10.1.0.4,56487,8587,6,I,D,NX,0,0,0,0","1733976098154,216.218.206.105,10.1.0.4,41536,6443,6,I,D,NX,0,0,0,0","1733976098509,79.110.62.68,10.1.0.4,47461,5578,6,I,D,NX,0,0,0,0","1733976101721,199.45.154.180,10.1.0.4,19145,1645,17,I,D,NX,0,0,0,0","1733976102809,79.110.62.162,10.1.0.4,45666,41369,6,I,D,NX,0,0,0,0","1733976106524,23.94.41.179,10.1.0.4,7025,5480,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:02:24.7518798Z"},{"category":"FlowLogFlowEvent","flowLogGUID":"dc407432-823e-4dbe-8d03-b33bb54b7dec","flowLogResourceID":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/NETWORKWATCHERRG/PROVIDERS/MICROSOFT.NETWORK/NETWORKWATCHERS/NETWORKWATCHER_WESTUS/FLOWLOGS/LABSVM1-VNET-AZURE-LABS-001-FLOWLOG","flowLogVersion":4,"flowRecords":{"flows":[{"aclID":"00000000-0000-0000-0000-000000000000","flowGroups":[{"flowTuples":["1733976142353,52.136.4.16,10.1.0.4,23456,23064,6,I,E,NX,1,128,0,0","1733976153201,40.87.164.0,10.1.0.4,23456,27459,6,I,E,NX,1,128,0,0","1733976195979,40.87.168.8,10.1.0.4,23456,29410,6,I,B,NX,0,0,0,0","1733976198617,40.87.164.0,10.1.0.4,23456,27462,6,I,E,NX,1,128,0,0"],"rule":"PlatformRule"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976155844,10.1.0.4,172.202.65.10,57999,443,6,O,B,NX,0,0,0,0","1733976161288,10.1.0.4,172.202.65.10,57999,443,6,O,E,NX,14,3027,26,27173"],"rule":"DefaultRule_AllowInternetOutBound"}]},{"aclID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/networkSecurityGroups/labsvm1-nsg","flowGroups":[{"flowTuples":["1733976151178,157.10.52.252,10.1.0.4,53542,3389,6,I,D,NX,0,0,0,0"],"rule":"UserRule_MicrosoftDefenderForCloud-JITRule_2017955537_88ABE006EA9C4130BF9065D4405169C7"},{"flowTuples":["1733976145510,79.110.62.68,10.1.0.4,47461,61523,6,I,D,NX,0,0,0,0","1733976150479,35.203.211.110,10.1.0.4,52556,24430,6,I,D,NX,0,0,0,0","1733976150601,92.255.85.51,10.1.0.4,51564,19623,6,I,D,NX,0,0,0,0","1733976153234,147.185.133.23,10.1.0.4,50577,46266,6,I,D,NX,0,0,0,0","1733976155135,194.180.49.219,10.1.0.4,58856,4650,6,I,D,NX,0,0,0,0","1733976156681,147.185.132.127,10.1.0.4,55295,45598,6,I,D,NX,0,0,0,0","1733976168973,147.185.133.93,10.1.0.4,52167,47512,6,I,D,NX,0,0,0,0","1733976170784,162.216.149.152,10.1.0.4,55770,54266,6,I,D,NX,0,0,0,0","1733976174863,206.168.35.187,10.1.0.4,24306,5061,6,I,D,NX,0,0,0,0","1733976176727,79.110.62.177,10.1.0.4,43008,12001,6,I,D,NX,0,0,0,0","1733976178593,13.64.193.117,10.1.0.4,51250,1337,6,I,D,NX,0,0,0,0","1733976179579,162.142.125.240,10.1.0.4,13609,52641,17,I,D,NX,0,0,0,0","1733976179751,162.216.149.27,10.1.0.4,49786,9556,6,I,D,NX,0,0,0,0","1733976180099,79.110.62.68,10.1.0.4,47461,61720,6,I,D,NX,0,0,0,0","1733976181004,35.203.210.156,10.1.0.4,56941,28444,6,I,D,NX,0,0,0,0","1733976181039,35.203.211.95,10.1.0.4,54031,9735,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976181557,18.194.211.253,10.1.0.4,55107,22,6,I,D,NX,0,0,0,0","1733976193409,193.68.89.3,10.1.0.4,49550,9055,6,I,D,NX,0,0,0,0","1733976196552,162.216.149.211,10.1.0.4,49834,9261,6,I,D,NX,0,0,0,0","1733976197528,205.210.31.184,10.1.0.4,55197,5909,6,I,D,NX,0,0,0,0","1733976198463,147.185.133.102,10.1.0.4,50085,23917,6,I,D,NX,0,0,0,0","1733976198737,79.110.62.68,10.1.0.4,47461,60741,6,I,D,NX,0,0,0,0","1733976199440,185.180.141.43,10.1.0.4,20730,6667,6,I,D,NX,0,0,0,0","1733976201177,194.180.49.220,10.1.0.4,58824,7092,6,I,D,NX,0,0,0,0"],"rule":"DefaultRule_DenyAllInBound"}]}]},"macAddress":"000D3A359573","operationName":"FlowLogFlowEvent","targetResourceID":"/subscriptions/c088dc46-d692-42ad-a4b6-9a542d28ad2a/resourceGroups/azure-labs-001/providers/Microsoft.Network/virtualNetworks/labsvm1-vnet","time":"2024-12-12T04:03:24.7776580Z"}]} \ No newline at end of file diff --git a/BlockBlobReader/tests/test_blobreader.py b/BlockBlobReader/tests/test_blobreader.py index 0378fa8c..6de43651 100644 --- a/BlockBlobReader/tests/test_blobreader.py +++ b/BlockBlobReader/tests/test_blobreader.py @@ -32,16 +32,18 @@ def setUpClass(cls): cls.test_storageaccount_name = "testsa%s" % (test_datetime_value) # Verify when Test Storage Account and template deployment are in different regions cls.test_storageAccountRegion = "Central US" - cls.log_type = os.environ.get("LOG_TYPE", "blob") + cls.FIXTURE_FILE = os.environ.get("FIXTURE_FILE", "blob_fixtures.json") + cls.log_type = cls.FIXTURE_FILE.split(".")[-1] - cls.test_container_name = "testcontainer-%s" % (datetime_value) if cls.log_type != "json" else "insights-logs-networksecuritygroupflowevent" + cls.test_container_name = cls.get_test_container_name() cls.test_filename_excluded_by_filter = "blockblob_test_filename_excluded_by_filter.blob" cls.test_filename_unsupported_extension = "blockblob_test.xml" # https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata # https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules # storageAccount 3-24 container 3-63 blobName 1-1024 maxDepth 63 if hierarchial namespace enabled # Verify maximum length path of storage location - folder_depth = str.join('/', choices(ascii_uppercase+digits, k=62)) + cls.MAX_FOLDER_DEPTH = int(os.environ.get("MAX_FOLDER_DEPTH", 62)) + folder_depth = str.join('/', choices(ascii_uppercase+digits, k=cls.MAX_FOLDER_DEPTH)) bigrandomfilename = str.join('', choices(ascii_uppercase+digits, k=(1024-10-len(folder_depth)))) # extension is appended later cls.test_filename = f"{folder_depth}/test{bigrandomfilename}" @@ -61,6 +63,15 @@ def setUpClass(cls): cls.create_resource_group( cls.resourcegroup_location, cls.resource_group_name) + @classmethod + def get_test_container_name(cls): + if cls.log_type != "json": + return "testcontainer-%s" % (datetime_value) + elif cls.FIXTURE_FILE == "blob_fixtures.json": + return "insights-logs-networksecuritygroupflowevent" + else: + return "insights-logs-flowlogflowevent" + def test_01_pipeline(self): self.deploy_template() self.assertTrue(self.resource_group_exists(self.resource_group_name)) @@ -102,7 +113,7 @@ def get_full_testlog_file_name(self): return expected_filename def test_03_func_logs(self): - self.logger.info("inserting mock %s data in BlobStorage" % self.log_type) + self.logger.info("inserting mock %s data in BlobStorage" % self.FIXTURE_FILE) if self.log_type in ("csv", "log", "blob"): self.insert_mock_logs_in_BlobStorage(self.log_type) else: @@ -134,20 +145,23 @@ def test_03_func_logs(self): f"No success message found in {azurefunction} azure function logs") self.assertFalse(self.filter_logs(captured_output, 'severityLevel', '3'), - f"Error messages found in {azurefunction} azure function logs") + f"Error messages found in {azurefunction} azure function logs: {captured_output}") self.assertFalse(self.filter_logs(captured_output, 'severityLevel', '2'), - f"Warning messages found in {azurefunction} azure function logs") + f"Warning messages found in {azurefunction} azure function logs: {captured_output}") self.logger.info("fetching mock data count from sumo") query = f'_sourceCategory="{self.source_category}" | count by _sourceName, _sourceHost' relative_time_in_minutes = 30 expected_record_count = { - "blob": 15, - "log": 10, - "json": 153, - "csv": 12 + "blob_fixtures.blob": 15, + "blob_fixtures.log": 10, + "blob_fixtures.json": 153, + "blob_fixtures_subnetflowlogs.json": 68, + "blob_fixtures_vnetflowlogs.json": 166, + "blob_fixtures_networkinterfaceflowlogs.json": 98, + "blob_fixtures.csv": 12 } record_count = record_excluded_by_filter_count = record_unsupported_extension_count = None source_host = source_name = "" @@ -162,8 +176,8 @@ def test_03_func_logs(self): except Exception as err: self.logger.info(f"Error in fetching sumo query results {err}") - self.assertTrue(record_count == expected_record_count.get(self.log_type), - f"block blob file's record count: {record_count} differs from expected count {expected_record_count.get(self.log_type)} in sumo '{self.source_category}'") + self.assertTrue(record_count == expected_record_count.get(self.FIXTURE_FILE), + f"block blob file's record count: {record_count} differs from expected count {expected_record_count.get(self.FIXTURE_FILE)} in sumo '{self.source_category}'") # Verify Filter Prefix field self.assertTrue(record_excluded_by_filter_count == 0, @@ -184,7 +198,7 @@ def test_03_func_logs(self): def upload_message_in_service_bus(self): file_ext = f".{self.log_type}" test_filename = self.test_filename + file_ext - with open(f"blob_fixtures{file_ext}", "r") as fp: + with open(self.FIXTURE_FILE, "r") as fp: file_size = len(fp.read()) triggerData = { @@ -243,11 +257,14 @@ def subtest_DLQ_func_logs(self): self.assertTrue(self.filter_logs(captured_output, 'message', successful_dlq_delete_message), f"No success delete dlq message found in {azurefunction} azure function logs") - self.assertFalse(self.filter_logs(captured_output, 'severityLevel', '3'), - f"Error messages found in {azurefunction} azure function logs") + error_messages = self.filter_logs(captured_output, 'severityLevel', '3') + self.assertFalse(error_messages, + f"Error messages found in {azurefunction} azure function logs: {error_messages}") + + warning_messages = self.filter_logs(captured_output, 'severityLevel', '2') + self.assertFalse(warning_messages, + f"Warning messages found in {azurefunction} azure function logs: {warning_messages}") - self.assertFalse(self.filter_logs(captured_output, 'severityLevel', '2'), - f"Warning messages found in {azurefunction} azure function logs") def get_random_name(self, length=32): return str(uuid.uuid4()) @@ -307,7 +324,7 @@ def get_current_blocks(self, test_container_name, test_filename): return blocks def get_json_data(self): - with open("blob_fixtures.json") as fp: + with open(self.FIXTURE_FILE) as fp: json_data = json.load(fp)["records"] return [json_data[:2], json_data[2:5], json_data[5:7], json_data[7:]] @@ -344,7 +361,7 @@ def insert_mock_json_in_BlobStorage(self): def get_csv_data(self): all_lines = [] - with open("blob_fixtures.csv") as logfile: + with open(self.FIXTURE_FILE) as logfile: all_lines = logfile.readlines() return [all_lines[:2], all_lines[2:5], all_lines[5:7]] + self.get_chunks(all_lines[7:], 2) @@ -353,13 +370,13 @@ def get_chunks(self, l, s): def get_log_data(self): all_lines = [] - with open("blob_fixtures.log") as logfile: + with open(self.FIXTURE_FILE) as logfile: all_lines = logfile.readlines() return [all_lines[:2], all_lines[2:5], all_lines[5:7], all_lines[7:]] def get_blob_formatted_data(self): all_lines = [] - with open("blob_fixtures.blob") as logfile: + with open(self.FIXTURE_FILE) as logfile: all_lines = logfile.readlines() return [all_lines[:2], all_lines[2:5], all_lines[5:7]] + self.get_chunks(all_lines[7:], 2) From a0a0293a2c3c08d391f96184a50699698fcaceea Mon Sep 17 00:00:00 2001 From: Himanshu Pal Date: Wed, 1 Jan 2025 12:43:15 +0530 Subject: [PATCH 3/3] added bash script for running tests --- AppendBlobReader/README.md | 17 ++++++++------- .../tests/run_integration_test.sh | 15 +++++++++++++ BlockBlobReader/README.md | 20 ++++++++++-------- BlockBlobReader/tests/run_integration_test.sh | 21 +++++++++++++++++++ EventHubs/README.md | 12 +++++++++++ EventHubs/tests/run_integration_test.sh | 15 +++++++++++++ 6 files changed, 83 insertions(+), 17 deletions(-) create mode 100644 AppendBlobReader/tests/run_integration_test.sh create mode 100755 BlockBlobReader/tests/run_integration_test.sh create mode 100644 EventHubs/tests/run_integration_test.sh diff --git a/AppendBlobReader/README.md b/AppendBlobReader/README.md index 37d20271..e492a9fe 100644 --- a/AppendBlobReader/README.md +++ b/AppendBlobReader/README.md @@ -35,16 +35,17 @@ This command copies required files in `AppendBlobReader/target` directory Integrations tests are in `AppendBlobReader/tests` folder and unit tests are in sumo-`function-utils/tests` folder +Modify the run_integration_test.sh file with below parameters ```console -export AZURE_SUBSCRIPTION_ID=`` -export AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application` -export AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials` -export AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant` -export AZURE_DEFAULT_REGION=`eastus` -export SUMO_ACCESS_ID=`` -export SUMO_ACCESS_KEY=`` -export SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security` +AZURE_SUBSCRIPTION_ID=`` +AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application` +AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials` +AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant` +AZURE_DEFAULT_REGION=`eastus` +SUMO_ACCESS_ID=`` +SUMO_ACCESS_KEY=`` +SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security` ``` Execute below command under `AppendBlobReader/tests` directory diff --git a/AppendBlobReader/tests/run_integration_test.sh b/AppendBlobReader/tests/run_integration_test.sh new file mode 100644 index 00000000..0d7d23e8 --- /dev/null +++ b/AppendBlobReader/tests/run_integration_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +export AZURE_SUBSCRIPTION_ID="" +# application id +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" +export AZURE_TENANT_ID="" +export AZURE_DEFAULT_REGION="eastus" +export SUMO_ACCESS_ID="" +export SUMO_ACCESS_KEY="" +export SUMO_DEPLOYMENT="us1" +export TEMPLATE_NAME="appendblobreaderdeploy.json" +python test_appendblobreader.py +# For deleting leftover resources in case of failures +# python ~/git/sumologic-azure-function/deletetestresourcegroups.py \ No newline at end of file diff --git a/BlockBlobReader/README.md b/BlockBlobReader/README.md index 5e5dcadd..7cdc4d0e 100644 --- a/BlockBlobReader/README.md +++ b/BlockBlobReader/README.md @@ -37,19 +37,21 @@ Integrations tests are in `BlockBlobReader/tests` folder and unit tests are in ` Integrations tests are in `BlockBlobReader/tests` folder and unit tests are in sumo-`function-utils/tests` folder -```console +Modify the run_integration_test.sh file with below parameters -export AZURE_SUBSCRIPTION_ID=`` -export AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application` -export AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials` -export AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant` -export AZURE_DEFAULT_REGION=`eastus` -export SUMO_ACCESS_ID=`` -export SUMO_ACCESS_KEY=`` -export SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security` +```console +AZURE_SUBSCRIPTION_ID=`` +AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application` +AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials` +AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant` +AZURE_DEFAULT_REGION=`eastus` +SUMO_ACCESS_ID=`` +SUMO_ACCESS_KEY=`` +SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security` ``` + Execute below command under `BlockBlobReader/tests` directory `python test_blobreader.py` diff --git a/BlockBlobReader/tests/run_integration_test.sh b/BlockBlobReader/tests/run_integration_test.sh new file mode 100755 index 00000000..93b562bd --- /dev/null +++ b/BlockBlobReader/tests/run_integration_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +export AZURE_SUBSCRIPTION_ID="" +# application id +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" +export AZURE_TENANT_ID="" +export AZURE_DEFAULT_REGION="eastus" +export SUMO_ACCESS_ID="" +export SUMO_ACCESS_KEY="" +export SUMO_DEPLOYMENT="us1" +export TEMPLATE_NAME="blobreaderdeploy.json" +# export FIXTURE_FILE="blob_fixtures.json" +export FIXTURE_FILE="blob_fixtures_vnetflowlogs.json" +# export FIXTURE_FILE="blob_fixtures_subnetflowlogs.json" +# export FIXTURE_FILE="blob_fixtures_networkinterfaceflowlogs.json" +export MAX_FOLDER_DEPTH=1 +# export TEMPLATE_NAME="blobreaderdeploywithPremiumPlan.json" +# export TEMPLATE_NAME="blobreaderzipdeploy.json" +python test_blobreader.py +# python ~/git/sumologic-azure-function/deletetestresourcegroups.py diff --git a/EventHubs/README.md b/EventHubs/README.md index a211f740..ca7ccf7c 100644 --- a/EventHubs/README.md +++ b/EventHubs/README.md @@ -24,3 +24,15 @@ This command copies required files in two directories logs_build(used for activi Integrations tests are in EventHubs/tests folder and unit tests are in sumo-function-utils/tests folder +Modify the run_integration_test.sh file with below parameters +```console + +AZURE_SUBSCRIPTION_ID=`` +AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application` +AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials` +AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant` +AZURE_DEFAULT_REGION=`eastus` +SUMO_ACCESS_ID=`` +SUMO_ACCESS_KEY=`` +SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security` +``` diff --git a/EventHubs/tests/run_integration_test.sh b/EventHubs/tests/run_integration_test.sh new file mode 100644 index 00000000..33e054a0 --- /dev/null +++ b/EventHubs/tests/run_integration_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +export AZURE_SUBSCRIPTION_ID="" +# application id +export AZURE_CLIENT_ID="" +export AZURE_CLIENT_SECRET="" +export AZURE_TENANT_ID="" +export AZURE_DEFAULT_REGION="eastus" +export SUMO_ACCESS_ID="" +export SUMO_ACCESS_KEY="" +export SUMO_DEPLOYMENT="us1" +export TEMPLATE_NAME="azuredeploy_metrics.json" +python test_eventhub_metrics.py +# For deleting leftover resources in case of failures +# python ~/git/sumologic-azure-function/deletetestresourcegroups.py \ No newline at end of file