TERR-361

- updated YAML docker-compose to use infinity datasource grafana plugin - reconfigured config.info.getGoogleSheetInfo fn to delegate all URL computation to e2e config - refactored e2e.config fileId out and replaced with flowSheetUrl prop - refactored folderDashboardInit.getFolderDashboardTargets to utilize infinity datasource plugin and reconfigure type to TSV and data flow URL. - simplified use of getGoogleSheetInfo in grafana-api.createDatasource - updated mock.panel and networkMapPanel JSON to use infinity plugin instead of CSV data source. - added ITarget interface to plugin-def - updated INetworkMapPanel interface to use a Partial<IDataSource> interface for datasource prop - added props url and queryType in object parameter when invoking getFolderDashboardTargets. - added props url and queryType to INetworkPanelParams
esnet · Jun 8, 2024 · 517d3c7 · 517d3c7
1 parent 1f8d2ad
commit 517d3c7
Show file tree

Hide file tree

Showing 10 changed files with 132 additions and 30 deletions.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -1,7 +1,7 @@
 services:
   grafana:
     environment:
-      - GF_INSTALL_PLUGINS=marcusolsson-csv-datasource
+      - GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource
     container_name: 'esnet-networkmap-panel'
     build:
       context: ./.config

diff --git a/e2e/config.info.ts b/e2e/config.info.ts
@@ -2,7 +2,7 @@ import dockerInfo from '../e2e/grafana-docker.json';
 import e2eConfig from '../e2e/e2e.config.json';
 
 const moduleName = 'config.info';
-const { fileId } = e2eConfig;
+const { flowSheetUrl } = e2eConfig;
 const targetGrafanaInstanceName = e2eConfig.grafanaInstanceName || "grafana";
 const grafanaInfo = (dockerInfo as Array<any>).find(nfo => nfo.Name == `/${targetGrafanaInstanceName}`);
 if (!grafanaInfo) {
@@ -12,8 +12,8 @@ if (!grafanaInfo) {
 }
 const { IPAddress, Ports } = grafanaInfo?.NetworkSettings;
 
-export const getGoogleSheetInfo = (fileId: string): string => {
-  const flowsUrl = `https://docs.google.com/spreadsheets/d/${fileId}/gviz/tq?tqx=out:csv`;
+export const getGoogleSheetInfo = (): string => {
+  const flowsUrl = flowSheetUrl;
 
   return flowsUrl;
 };

diff --git a/e2e/e2e.config.json b/e2e/e2e.config.json
@@ -3,6 +3,6 @@
     "targetDashboard": "network-map-test-dashboard",
     "targetFolder": "network-map-test-folder",
     "grafanaInstanceName": "esnet-networkmap-panel",
-    "fileId": "1K_nZcu4yzPXBuOR3nO8NkbSCxMnvWtu37H9cGagkQgc",
+    "flowSheetUrl": "https://docs.google.com/spreadsheets/d/e/2PACX-1vQn18dEVlFjvL3arvbiaOZIKkLseVSIXFg9Gw3Qp4rY2KruDvAZ0FfYMylt31Ia3Nx8Gxm08alIMmtW/pub?gid=1087577778&single=true&output=tsv",
     "topologySheetUrl": "https://docs.google.com/spreadsheets/d/e/2PACX-1vQn18dEVlFjvL3arvbiaOZIKkLseVSIXFg9Gw3Qp4rY2KruDvAZ0FfYMylt31Ia3Nx8Gxm08alIMmtW/pub?gid=261150740&single=true&output=csv"
 }
diff --git a/e2e/folderDashboardInit.ts b/e2e/folderDashboardInit.ts
@@ -1,11 +1,11 @@
 import { createDashboard, createFolder, getCurrentOrg, getDashboard, updateDashboard } from './grafana-api';
-import { getHostInfo, getGoogleSheetInfo } from './config.info';
+import { getHostInfo } from './config.info';
 import credentials from '../playwright/.auth/credentials.json';
 import e2eConfig from './e2e.config.json';
 import mockPanelImport from './mock.panel.json';
 import { ITargets } from './interfaces/Targets.interface';
 import { INetworkPanelParams } from './interfaces/PanelParams.interface';
-import { IDashboardResponse, INetworkMapPanel, IOrganization, IPanel } from './plugin-def';
+import { IDashboardResponse, INetworkMapPanel, IOrganization, IPanel, ITarget } from './plugin-def';
 
 let targetFolderUid;
 let targetDashboardUid;
@@ -53,8 +53,10 @@ export const getFolderDashboardTargets = async (params?: INetworkPanelParams): P
       title: targetDashboardTitle
     });
 
-    // store targetDashboardUid for later
+    // configure query
     const targetDashboard = JSON.parse(targetDashboardJsonStr);
+
+    // store targetDashboardUid for later
     targetDashboardUid = targetDashboard.uid;
     dashboardJsonStr = await getDashboard(targetDashboardUid);
   } else if (dashboardSearchResponseJson.length === 1) {
@@ -97,6 +99,75 @@ export const getFolderDashboardTargets = async (params?: INetworkPanelParams): P
 
   // update dashboard values data structure w/ topology and traffic flow data if specified into panel
   if (params) {
+    // query settings
+    if (params.url || params.queryType) {
+      let updatedTargetPartial;
+      const baseTarget = {
+        columns: [],
+        filters: [],
+        format: "table",
+        global_query_id: "",
+        root_selector: "",
+        skipRows: 0
+      }
+      if (Array.isArray(dashboardInfo.dashboard.targets)) {
+        updatedTargetPartial = {
+          ...baseTarget,
+          ...dashboardInfo.dashboard.targets[0],
+        };
+      } else {
+        updatedTargetPartial = baseTarget;
+      }
+
+      if (params.url) {
+        updatedTargetPartial.url = params.url;
+      }
+      if (params.queryType) {
+        updatedTargetPartial.type = params.queryType;
+      }
+
+      const panelCount = dashboardInfo.dashboard.panels.length;
+      const panelIdx = dashboardInfo.dashboard.panels.findIndex(panel => panel.type === "esnet-network-mappanel");
+      let panelsArr: IPanel[] = [];
+
+      // iterate through panels in dashboard
+      for (let currPanelIdx = 0; currPanelIdx < panelCount; currPanelIdx++) {
+
+        // in each panel's targets find all targets with infinity datasource
+        const currPanel = dashboardInfo.dashboard.panels[currPanelIdx];
+        const targetCount = currPanel.targets?.length ?? 0;
+        const panelTargetIdxs: number[] | undefined = currPanel.targets?.reduce((acc: number[], target: ITarget, targetIdx: number) => {
+          if (target.datasource.type === "yesoreyeram-infinity-datasource") {
+            acc.push(targetIdx);
+          }
+          return acc;
+        }, [] as number[]);
+
+        // iterate through all infinity targets, updating those with param url and type...
+        const targetArr: ITarget[] = [];
+        for (let currTargetIdx = 0; currTargetIdx < targetCount; currTargetIdx++) {
+          let currTarget = currPanel.targets![currTargetIdx];
+          if (panelTargetIdxs && panelTargetIdxs.includes(currPanelIdx)) {
+            currTarget = {
+              ...currTarget,
+              ...updatedTargetPartial,
+            };
+          }
+          targetArr.push(currTarget);
+        }
+
+        // set panel's targets
+        currPanel.targets = targetArr;
+        // add panel to results
+        panelsArr.push(currPanel);
+      }
+
+      // update container for dashboard prior to API invocation
+      dashboardInfo.dashboard = {
+        ...dashboardInfo.dashboard,
+        panels: panelsArr
+      };
+    }
     // topology
     if (params.topology) {
       const { panels } = dashboardInfo.dashboard || {panels: currentPanels};
@@ -109,7 +180,7 @@ export const getFolderDashboardTargets = async (params?: INetworkPanelParams): P
           // traffic flow data from a data source
           if (params.uid) {
             datasource = {
-              type: "marcusolsson-csv-datasource",
+              type: "yesoreyeram-infinity-datasource",
               uid: params.uid
             }
           }

diff --git a/e2e/grafana-api.ts b/e2e/grafana-api.ts
@@ -22,13 +22,10 @@ const DEFAULT_DATASOURCE_NAME = "network-traffic-flow";
  * @param {boolean} forceCreate
  * @returns
  */
-export const createDatasource = async (fileId?: string, forceCreate = false): Promise<IDataSource> => {
+export const createDatasource = async (forceCreate = false): Promise<IDataSource> => {
   const fnName = "folderDashboardInit.initCSVDatasource";
   const { basicAuthHeader, protocolHostPort } = await getHostInfo(credentials);
-  let dataFlowUrl: string | null = null;
-  if (fileId) {
-    dataFlowUrl = getGoogleSheetInfo(fileId);
-  }
+  let dataFlowUrl: string | null = getGoogleSheetInfo();
 
   let resultDataSource;
   try {
@@ -46,27 +43,24 @@ export const createDatasource = async (fileId?: string, forceCreate = false): Pr
       redirect: "follow"
     });
 
-    if (dataSrcCheckResponse.ok) {
+    if (dataSrcCheckResponse.ok && !forceCreate) {
       // if exists, return info
       resultDataSource = await dataSrcCheckResponse.json();
-    } else if (!dataSrcCheckResponse.ok && dataSrcCheckResponse.status === 404) {
-      // if does not exist, create it
+    } else if (!dataSrcCheckResponse.ok && dataSrcCheckResponse.status === 404 || forceCreate) {
+      // if does not exist or forced to create, create it
       const inObj = {
         name: DEFAULT_DATASOURCE_NAME,
-        type: "marcusolsson-csv-datasource",
+        type: "yesoreyeram-infinity-datasource",
         access: "proxy"
       };
-      // initialize with data url
-      if (dataFlowUrl) {
-        inObj["url"] = dataFlowUrl;
-      }
+      // no need to use URL, that is specified at the dashboard level
+      // post url to grafana API for data source config
       const dataSrcCreateResponse: Response = await fetch(`${protocolHostPort}/api/datasources`, {
         method: "POST",
         headers: {
           ...basicAuthHeader,
           ...jsonHeaders
         },
-        redirect: 'follow',
         body: JSON.stringify(inObj)
       });
       const jsonResponse = await dataSrcCreateResponse.json();

diff --git a/e2e/interfaces/PanelParams.interface.ts b/e2e/interfaces/PanelParams.interface.ts
@@ -24,4 +24,13 @@ export interface INetworkPanelParams {
      * The uid of the data source for the network map panel overall
      */
     uid?: string;
+    /**
+     * The URL to configure for the panel's query.
+     * @var
+     */
+    url?: string;
+    /**
+     * The query type to configure for the panel's query.
+     */
+    queryType?: string;
 }
diff --git a/e2e/mock.panel.json b/e2e/mock.panel.json
@@ -1,6 +1,6 @@
 {
     "datasource": {
-        "type": "marcusolsson-csv-datasource",
+        "type": "yesoreyeram-infinity-datasource",
         "uid": "bdk95xlce6i9sc"
     },
     "fieldConfig": {
@@ -134,7 +134,7 @@
     "targets": [
         {
             "datasource": {
-                "type": "marcusolsson-csv-datasource",
+                "type": "yesoreyeram-infinity-datasource",
                 "uid": "bdk95xlce6i9sc"
             },
             "decimalSeparator": ".",

diff --git a/e2e/networkMapPanel.json b/e2e/networkMapPanel.json
@@ -1,6 +1,6 @@
 {
     "datasource": {
-        "type": "marcusolsson-csv-datasource",
+        "type": "yesoreyeram-infinity-datasource",
         "uid": "nU_iqSsVk"
     },
     "fieldConfig": {
@@ -129,7 +129,7 @@
     "targets": [
         {
             "datasource": {
-                "type": "marcusolsson-csv-datasource",
+                "type": "yesoreyeram-infinity-datasource",
                 "uid": "nU_iqSsVk"
             },
             "decimalSeparator": ".",

diff --git a/e2e/plugin-def.ts b/e2e/plugin-def.ts
@@ -1,6 +1,7 @@
 import { PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, TestType, test as base } from '@playwright/test';
 import { ITargets } from './interfaces/Targets.interface';
 import { MapOptions } from '../src/types';
+import { IDataSource } from './interfaces/DataSource.interface';
 
 export type PluginTestOptions = {
   targets: ITargets;
@@ -19,14 +20,15 @@ export interface IPanel {
   id: number;
   mode: string;
   content: string;
+  targets?: ITarget[];
 }
 
 /**
  * The JSON model for Network Map Panel
  */
 export interface INetworkMapPanel extends IPanel {
   options: MapOptions;
-  datasource: { type: string; uid: string; }
+  datasource: Partial<IDataSource>;
 }
 
 export type TimeRange = { to: string; from: string; };
@@ -91,6 +93,30 @@ export interface IDashboard {
   schemaVersion: number;
   version: number;
   panels: IPanel[];
+  targets: ITarget[];
+}
+
+export interface ITarget {
+  columns?: string[];
+  datasource: Partial<IDataSource>;
+  decimalSeparator: string;
+  delimiter: string;
+  filters?: any[];
+  format?: string;
+  global_query_id?: string;
+  header: boolean;
+  ignoreUnknown: boolean;
+  refId: string;
+  root_selector?: string;
+  schema: any[];
+  skipRows?: number;
+  source?: string;
+  type?: string;
+  url?: string;
+  url_options?: {
+    data: string;
+    method: string;
+  }
 }
 
 export interface IOrganization {

diff --git a/e2e/plugin.spec.ts b/e2e/plugin.spec.ts
@@ -1,6 +1,6 @@
 import { expect, Page } from '@playwright/test';
 import testIds from '../src/constants';
-import { topologySheetUrl as topologyUrl, fileId } from '../e2e/e2e.config.json';
+import { topologySheetUrl as topologyUrl, flowSheetUrl as flowUrl } from '../e2e/e2e.config.json';
 import { getHostInfo } from './config.info';
 import credentials from '../playwright/.auth/credentials.json';
 import { ITargets } from './interfaces/Targets.interface';
@@ -42,7 +42,7 @@ pluginTest.describe("plugin testing", () => {
   pluginTest.use({
     targets: async ({}, use) => {
       // setup data source
-      const dataSource = await createDatasource(fileId);
+      const dataSource = await createDatasource();
 
       // get topology
       const topologyResponse: Response = await fetch(topologyUrl, {
@@ -54,6 +54,8 @@ pluginTest.describe("plugin testing", () => {
       const topology = JSON.parse(topologyResponseUnescaped);
       // setup dashboard, including topology data from datasource uid
       const newFixtureObj = await getFolderDashboardTargets({
+        url: flowUrl,
+        queryType: 'tsv',
         topology,
         uid: dataSource.uid
       });