Merge pull request #73 from SumoLogic/azure-metrics-upgrade-v2

Azure metrics collection upgrade to v2
SumoLogic · Jan 26, 2024 · 7e8e25b · 7e8e25b
2 parents a864c0f + 05ce87e
commit 7e8e25b
Show file tree

Hide file tree

Showing 24 changed files with 594 additions and 689 deletions.
diff --git a/.github/workflows/arm-template-test.yml b/.github/workflows/arm-template-test.yml
@@ -0,0 +1,35 @@
+name: "ARM template tests"
+
+on: [pull_request]
+
+jobs:
+  best-practices-test:
+    name: "ARM template tests using arm-ttk"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Test ARM Templates using arm-ttk
+        id: filestest
+        uses: aliencube/[email protected]
+        with:
+          path: EventHubs/src/azuredeploy_metrics.json
+
+  security-test:
+    name: "ARM template tests using checkov"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Test ARM Templates using checkov
+        id: checkov
+        uses: bridgecrewio/checkov-action@v12
+        with:
+          file: EventHubs/src/azuredeploy_metrics.json
+          skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35
+          quiet: false
+          framework: arm
+          output_format: cli
+          output_bc_ids: true
diff --git a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumoclient.js b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumoclient.js
@@ -125,11 +125,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
     var self = this;
     let curOptions = Object.assign({},this.options);
 
-    //this.context.log("Flush buffer for metaKey:"+metaKey);
+    this.context.debug("Flush buffer for metaKey:"+metaKey);
 
     function httpSend(messageArray,data) {
         return new Promise( (resolve,reject) => {
-            //self.context.log("Inside HTTP Send");
             var req = https.request(curOptions, function (res) {
                 var body = '';
                 res.setEncoding('utf8');
@@ -155,7 +154,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
             });
             req.write(data);
             req.end();
-       });
+        });
     }
 
     if (targetBuffer) {
@@ -176,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
 
             return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
                 if (!e)  {
-                    self.context.log("gzip successful");
+                    self.context.debug("gzip successful");
                     return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
                             .then(()=> {
-                        self.context.log("Succesfully sent " + self.messagesSent + " messages to Sumo");
+                        self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
                         self.success_callback(self.context);}
                         )
                     .catch((err) => {
@@ -194,7 +193,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
                     self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
                     self.failure_callback(msgArray,self.context);
                 }
-                });
+            });
         }  else {
             //self.context.log('Send raw data to Sumo');
             return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
@@ -253,7 +252,7 @@ function FlushFailureHandler (messageArray,ctx) {
 };
 
 /**
- * Default built-in callback function to handle successful sents. It simply logs a success message
+ * Default built-in callback function to handle successful sent. It simply logs a success message
  * @param ctx is a context variable that supports a log method
  * @constructor
  */

diff --git a/BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumometricclient.js b/BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumometricclient.js
@@ -2,9 +2,9 @@
 /**
  * Created by duc on 6/30/17. This is a client for metric
  */
-var https = require('https');
-var zlib= require('zlib');
-var url = require('url');
+var https = require('node:https');
+var zlib = require('node:zlib');
+var url = require('node:url');
 
 var sumoclient = require('./sumoclient');
 var bucket = require('./messagebucket');
@@ -119,7 +119,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
     var self = this;
     let curOptions = Object.assign({},this.options);
 
-    this.context.log("Flush METRIC buffer for metaKey:"+metaKey);
+    this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
 
     function httpSend(messageArray,data) {
 
@@ -170,11 +170,12 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
         if (curOptions.compress_data) {
             curOptions.headers['Content-Encoding'] = 'gzip';
 
-            zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
+            return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
                 if (!e)  {
+                    self.context.debug("gzip successful");
                     sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
                         .then(()=> {
-                        //self.context.log("Succesfully sent to Sumo after "+self.MaxAttempts);
+                        self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
                         self.success_callback(self.context);}
                 )
                 .catch((err) => {
@@ -184,20 +185,21 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
                     self.failure_callback(msgArray,self.context);
                 });
                 } else {
+                    self.messagesFailed += msgArray.length;
                     self.messagesAttempted += msgArray.length;
                     self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
                     self.failure_callback(msgArray,self.context);
                 }
             });
         }  else {
             //self.context.log('Send raw data to Sumo');
-            sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
+            return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
                 .then(()=> { self.success_callback(self.context);})
-        .catch((err) => {
-            self.messagesFailed += msgArray.length;
-            self.messagesAttempted += msgArray.length;
-            self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
-            self.failure_callback(msgArray,self.context);
+            .catch((err) => {
+                self.messagesFailed += msgArray.length;
+                self.messagesAttempted += msgArray.length;
+                self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
+                self.failure_callback(msgArray,self.context);
         });
         }
     }
@@ -215,7 +217,7 @@ function FlushFailureHandler (messageArray,ctx) {
 };
 
 /**
- * Default built-in callback function to handle successful sents. It simply logs a success message
+ * Default built-in callback function to handle successful sent. It simply logs a success message
  * @param ctx is a context variable that supports a log method
  * @constructor
  */
@@ -227,5 +229,4 @@ module.exports = {
     SumoMetricClient:SumoMetricClient,
     FlushFailureHandler:FlushFailureHandler,
     DefaultSuccessHandler:DefaultSuccessHandler
-}
-
+}
diff --git a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumoclient.js b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumoclient.js
@@ -125,11 +125,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
     var self = this;
     let curOptions = Object.assign({},this.options);
 
-    //this.context.log("Flush buffer for metaKey:"+metaKey);
+    this.context.debug("Flush buffer for metaKey:"+metaKey);
 
     function httpSend(messageArray,data) {
         return new Promise( (resolve,reject) => {
-            //self.context.log("Inside HTTP Send");
             var req = https.request(curOptions, function (res) {
                 var body = '';
                 res.setEncoding('utf8');
@@ -155,7 +154,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
             });
             req.write(data);
             req.end();
-       });
+        });
     }
 
     if (targetBuffer) {
@@ -176,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
 
             return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
                 if (!e)  {
-                    self.context.log("gzip successful");
+                    self.context.debug("gzip successful");
                     return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
                             .then(()=> {
-                        self.context.log("Succesfully sent " + self.messagesSent + " messages to Sumo");
+                        self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
                         self.success_callback(self.context);}
                         )
                     .catch((err) => {
@@ -194,7 +193,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
                     self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
                     self.failure_callback(msgArray,self.context);
                 }
-                });
+            });
         }  else {
             //self.context.log('Send raw data to Sumo');
             return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
@@ -253,7 +252,7 @@ function FlushFailureHandler (messageArray,ctx) {
 };
 
 /**
- * Default built-in callback function to handle successful sents. It simply logs a success message
+ * Default built-in callback function to handle successful sent. It simply logs a success message
  * @param ctx is a context variable that supports a log method
  * @constructor
  */

diff --git a/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumometricclient.js b/BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumometricclient.js
@@ -2,9 +2,9 @@
 /**
  * Created by duc on 6/30/17. This is a client for metric
  */
-var https = require('https');
-var zlib= require('zlib');
-var url = require('url');
+var https = require('node:https');
+var zlib = require('node:zlib');
+var url = require('node:url');
 
 var sumoclient = require('./sumoclient');
 var bucket = require('./messagebucket');
@@ -119,7 +119,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
     var self = this;
     let curOptions = Object.assign({},this.options);
 
-    this.context.log("Flush METRIC buffer for metaKey:"+metaKey);
+    this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
 
     function httpSend(messageArray,data) {
 
@@ -170,11 +170,12 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
         if (curOptions.compress_data) {
             curOptions.headers['Content-Encoding'] = 'gzip';
 
-            zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
+            return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
                 if (!e)  {
+                    self.context.debug("gzip successful");
                     sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
                         .then(()=> {
-                        //self.context.log("Succesfully sent to Sumo after "+self.MaxAttempts);
+                        self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
                         self.success_callback(self.context);}
                 )
                 .catch((err) => {
@@ -184,20 +185,21 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
                     self.failure_callback(msgArray,self.context);
                 });
                 } else {
+                    self.messagesFailed += msgArray.length;
                     self.messagesAttempted += msgArray.length;
                     self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
                     self.failure_callback(msgArray,self.context);
                 }
             });
         }  else {
             //self.context.log('Send raw data to Sumo');
-            sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
+            return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
                 .then(()=> { self.success_callback(self.context);})
-        .catch((err) => {
-            self.messagesFailed += msgArray.length;
-            self.messagesAttempted += msgArray.length;
-            self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
-            self.failure_callback(msgArray,self.context);
+            .catch((err) => {
+                self.messagesFailed += msgArray.length;
+                self.messagesAttempted += msgArray.length;
+                self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted  + ' messagesReceived: ' + self.messagesReceived);
+                self.failure_callback(msgArray,self.context);
         });
         }
     }
@@ -215,7 +217,7 @@ function FlushFailureHandler (messageArray,ctx) {
 };
 
 /**
- * Default built-in callback function to handle successful sents. It simply logs a success message
+ * Default built-in callback function to handle successful sent. It simply logs a success message
  * @param ctx is a context variable that supports a log method
  * @constructor
  */
@@ -227,5 +229,4 @@ module.exports = {
     SumoMetricClient:SumoMetricClient,
     FlushFailureHandler:FlushFailureHandler,
     DefaultSuccessHandler:DefaultSuccessHandler
-}
-
+}
diff --git a/EventHubs/requirements.txt b/EventHubs/requirements.txt
@@ -1,7 +1,10 @@
-requests==2.20.0
-azure-mgmt-resource==1.2.2
-azure-mgmt-eventhub==1.2.0
-azure-servicebus==0.21.1
-azure-cosmosdb-table==1.0.2
-azure-mgmt-storage==1.0.0
-
+requests==2.31.0
+azure-monitor-query==1.2.0
+azure-identity==1.15.0
+azure-core==1.29.6
+azure-mgmt-resource==23.0.1
+azure-mgmt-eventhub==11.0.0
+azure-eventhub==5.11.5
+azure-mgmt-storage==21.1.0
+azure-mgmt-loganalytics==12.0.0
+sumologic-sdk==0.1.15