remove records with grouping

Signed-off-by: Chenyang Ji <[email protected]>

public/pages/TopNQueries/TopNQueries.tsx

@@ -12,6 +12,7 @@ import QueryInsights from '../QueryInsights/QueryInsights';
 import Configuration from '../Configuration/Configuration';
 import QueryDetails from '../QueryDetails/QueryDetails';
 import { SearchQueryRecord } from '../../../types/types';
+import { removeQueryGroups } from '../../utils/utils';
 
 export const QUERY_INSIGHTS = '/queryInsights';
 export const CONFIGURATION = '/configuration';
@@ -158,7 +159,7 @@ const TopNQueries = ({
         const noDuplicates: SearchQueryRecord[] = Array.from(
           new Set(newQueries.map((item) => JSON.stringify(item)))
         ).map((item) => JSON.parse(item));
-        setQueries(noDuplicates);
+        setQueries(removeQueryGroups(noDuplicates));
       } catch (error) {
         console.error('Error retrieving queries:', error);
       } finally {

public/utils/utils.test.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { removeQueryGroups } from './utils';
+import { SearchQueryRecord } from '../../types/types';
+import { MockGroups, MockQueries } from '../../test/testUtils';
+
+const mockQueries = MockQueries();
+const mockGroups = MockGroups();
+describe('removeQueryGroups', () => {
+  it('should return an empty array when input is an empty array', () => {
+    const records: SearchQueryRecord[] = [];
+    const result = removeQueryGroups(records);
+    expect(result).toEqual([]);
+  });
+
+  it('should return only records where all measurements have count equal to 1 (native queries)', () => {
+    const records: SearchQueryRecord[] = [...mockQueries, ...mockGroups];
+    const result = removeQueryGroups(records);
+    expect(result).toEqual(mockQueries);
+  });
+
+  it('should exclude all records where all records are groups', () => {
+    const result = removeQueryGroups(mockGroups);
+    expect(result).toEqual([]);
+  });
+
+  it('should include records where all records are native queries', () => {
+    const result = removeQueryGroups(mockQueries);
+    expect(result).toEqual(mockQueries);
+  });
+
+  it('should handle empty records', () => {
+    const result = removeQueryGroups([]);
+    expect(result).toEqual([]);
+  });
+});

public/utils/utils.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SearchQueryRecord } from '../../types/types';
+
+export const removeQueryGroups = (records: SearchQueryRecord[]) => {
+  return records.filter((record: SearchQueryRecord) =>
+    Object.values(record.measurements).every((measurement) => measurement.count === 1)
+  );
+};

test/testUtils.ts

@@ -0,0 +1,192 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SearchQueryRecord } from '../types/types';
+
+export const MockQueries = (): SearchQueryRecord[] => {
+  return [
+    {
+      timestamp: 1726178995210,
+      task_resource_usages: [
+        {
+          action: 'indices:data/read/search[phase/query]',
+          taskId: 18809,
+          parentTaskId: 18808,
+          nodeId: 'Q36D2z_NRGKim6EZZMgi6A',
+          taskResourceUsage: {
+            cpu_time_in_nanos: 3612000,
+            memory_in_bytes: 123944,
+          },
+        },
+        {
+          action: 'indices:data/read/search',
+          taskId: 18808,
+          parentTaskId: -1,
+          nodeId: 'Q36D2z_NRGKim6EZZMgi6A',
+          taskResourceUsage: {
+            cpu_time_in_nanos: 1898000,
+            memory_in_bytes: 24176,
+          },
+        },
+      ],
+      source: {
+        query: {
+          bool: {
+            must: [
+              {
+                range: {
+                  timestamp: {
+                    from: 1726092595177,
+                    to: 1726178995177,
+                    include_lower: true,
+                    include_upper: true,
+                    boost: 1.0,
+                  },
+                },
+              },
+            ],
+            must_not: [
+              {
+                match: {
+                  indices: {
+                    query: 'top_queries*',
+                    operator: 'OR',
+                    prefix_length: 0,
+                    max_expansions: 50,
+                    fuzzy_transpositions: true,
+                    lenient: false,
+                    zero_terms_query: 'NONE',
+                    auto_generate_synonyms_phrase_query: true,
+                    boost: 1.0,
+                  },
+                },
+              },
+            ],
+            adjust_pure_negative: true,
+            boost: 1.0,
+          },
+        },
+      },
+      query_hashcode: '80a17984b847133b8bf5e7d5dfbfa96c',
+      phase_latency_map: {
+        expand: 0,
+        query: 5,
+        fetch: 0,
+      },
+      labels: {
+        'X-Opaque-Id': 'ae6c1170-5f98-47f4-b7fc-09ebcf574b81',
+      },
+      total_shards: 1,
+      search_type: 'query_then_fetch',
+      node_id: 'Q36D2z_NRGKim6EZZMgi6A',
+      indices: ['top_queries-2024.09.12'],
+      measurements: {
+        latency: {
+          number: 8,
+          count: 1,
+          aggregationType: 'NONE',
+        },
+        cpu: {
+          number: 5510000,
+          count: 1,
+          aggregationType: 'NONE',
+        },
+      },
+    },
+  ];
+};
+
+export const MockGroups = (): SearchQueryRecord[] => {
+  return [
+    {
+      timestamp: 1726178995210,
+      task_resource_usages: [
+        {
+          action: 'indices:data/read/search[phase/query]',
+          taskId: 18809,
+          parentTaskId: 18808,
+          nodeId: 'Q36D2z_NRGKim6EZZMgi6A',
+          taskResourceUsage: {
+            cpu_time_in_nanos: 3612000,
+            memory_in_bytes: 123944,
+          },
+        },
+        {
+          action: 'indices:data/read/search',
+          taskId: 18808,
+          parentTaskId: -1,
+          nodeId: 'Q36D2z_NRGKim6EZZMgi6A',
+          taskResourceUsage: {
+            cpu_time_in_nanos: 1898000,
+            memory_in_bytes: 24176,
+          },
+        },
+      ],
+      source: {
+        query: {
+          bool: {
+            must: [
+              {
+                range: {
+                  timestamp: {
+                    from: 1726092595177,
+                    to: 1726178995177,
+                    include_lower: true,
+                    include_upper: true,
+                    boost: 1.0,
+                  },
+                },
+              },
+            ],
+            must_not: [
+              {
+                match: {
+                  indices: {
+                    query: 'top_queries*',
+                    operator: 'OR',
+                    prefix_length: 0,
+                    max_expansions: 50,
+                    fuzzy_transpositions: true,
+                    lenient: false,
+                    zero_terms_query: 'NONE',
+                    auto_generate_synonyms_phrase_query: true,
+                    boost: 1.0,
+                  },
+                },
+              },
+            ],
+            adjust_pure_negative: true,
+            boost: 1.0,
+          },
+        },
+      },
+      query_hashcode: '80a17984b847133b8bf5e7d5dfbfa96c',
+      phase_latency_map: {
+        expand: 0,
+        query: 5,
+        fetch: 0,
+      },
+      labels: {
+        'X-Opaque-Id': 'ae6c1170-5f98-47f4-b7fc-09ebcf574b81',
+      },
+      total_shards: 1,
+      search_type: 'query_then_fetch',
+      node_id: 'Q36D2z_NRGKim6EZZMgi6A',
+      indices: ['top_queries-2024.09.12'],
+      measurements: {
+        latency: {
+          number: 8,
+          count: 5,
+          aggregationType: 'NONE',
+        },
+        cpu: {
+          number: 5510000,
+          count: 5,
+          aggregationType: 'NONE',
+        },
+      },
+    },
+  ];
+};

types/types.ts

@@ -3,8 +3,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { ISearchSource } from 'src/plugins/data/public';
-
 export interface SearchQueryRecord {
   timestamp: number;
   measurements: {
@@ -14,12 +12,13 @@ export interface SearchQueryRecord {
   };
   total_shards: number;
   node_id: string;
-  source: ISearchSource;
+  source: Record<string, any>;
   labels: Record<string, string>;
   search_type: string;
   indices: string[];
   phase_latency_map: PhaseLatencyMap;
   task_resource_usages: Task[];
+  query_hashcode: string;
 }
 
 export interface Measurement {