From 9aa0e1fe2f0bd65a716a59836e3b42efbe058587 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Wed, 6 Nov 2024 17:05:13 -0800 Subject: [PATCH] remove records with grouping Signed-off-by: Chenyang Ji --- public/pages/TopNQueries/TopNQueries.tsx | 3 +- public/utils/utils.test.ts | 38 +++++ public/utils/utils.ts | 12 ++ test/testUtils.ts | 192 +++++++++++++++++++++++ types/types.ts | 5 +- 5 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 public/utils/utils.test.ts create mode 100644 public/utils/utils.ts create mode 100644 test/testUtils.ts diff --git a/public/pages/TopNQueries/TopNQueries.tsx b/public/pages/TopNQueries/TopNQueries.tsx index db631dd..066e748 100644 --- a/public/pages/TopNQueries/TopNQueries.tsx +++ b/public/pages/TopNQueries/TopNQueries.tsx @@ -12,6 +12,7 @@ import QueryInsights from '../QueryInsights/QueryInsights'; import Configuration from '../Configuration/Configuration'; import QueryDetails from '../QueryDetails/QueryDetails'; import { SearchQueryRecord } from '../../../types/types'; +import { removeQueryGroups } from '../../utils/utils'; export const QUERY_INSIGHTS = '/queryInsights'; export const CONFIGURATION = '/configuration'; @@ -158,7 +159,7 @@ const TopNQueries = ({ const noDuplicates: SearchQueryRecord[] = Array.from( new Set(newQueries.map((item) => JSON.stringify(item))) ).map((item) => JSON.parse(item)); - setQueries(noDuplicates); + setQueries(removeQueryGroups(noDuplicates)); } catch (error) { console.error('Error retrieving queries:', error); } finally { diff --git a/public/utils/utils.test.ts b/public/utils/utils.test.ts new file mode 100644 index 0000000..de94aad --- /dev/null +++ b/public/utils/utils.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import { removeQueryGroups } from './utils'; +import { SearchQueryRecord } from '../../types/types'; +import { MockGroups, MockQueries } from '../../test/testUtils'; + +const mockQueries = MockQueries(); +const mockGroups = MockGroups(); +describe('removeQueryGroups', () => { + it('should return an empty array when input is an empty array', () => { + const records: SearchQueryRecord[] = []; + const result = removeQueryGroups(records); + expect(result).toEqual([]); + }); + + it('should return only records where all measurements have count equal to 1 (native queries)', () => { + const records: SearchQueryRecord[] = [...mockQueries, ...mockGroups]; + const result = removeQueryGroups(records); + expect(result).toEqual(mockQueries); + }); + + it('should exclude all records where all records are groups', () => { + const result = removeQueryGroups(mockGroups); + expect(result).toEqual([]); + }); + + it('should include records where all records are native queries', () => { + const result = removeQueryGroups(mockQueries); + expect(result).toEqual(mockQueries); + }); + + it('should handle empty records', () => { + const result = removeQueryGroups([]); + expect(result).toEqual([]); + }); +}); diff --git a/public/utils/utils.ts b/public/utils/utils.ts new file mode 100644 index 0000000..8922839 --- /dev/null +++ b/public/utils/utils.ts @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SearchQueryRecord } from '../../types/types'; + +export const removeQueryGroups = (records: SearchQueryRecord[]) => { + return records.filter((record: SearchQueryRecord) => + Object.values(record.measurements).every((measurement) => measurement.count === 1) + ); +}; diff --git a/test/testUtils.ts b/test/testUtils.ts new file mode 100644 index 0000000..6cbc427 --- /dev/null +++ b/test/testUtils.ts @@ -0,0 +1,192 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SearchQueryRecord } from '../types/types'; + +export const MockQueries = (): SearchQueryRecord[] => { + return [ + { + timestamp: 1726178995210, + task_resource_usages: [ + { + action: 'indices:data/read/search[phase/query]', + taskId: 18809, + parentTaskId: 18808, + nodeId: 'Q36D2z_NRGKim6EZZMgi6A', + taskResourceUsage: { + cpu_time_in_nanos: 3612000, + memory_in_bytes: 123944, + }, + }, + { + action: 'indices:data/read/search', + taskId: 18808, + parentTaskId: -1, + nodeId: 'Q36D2z_NRGKim6EZZMgi6A', + taskResourceUsage: { + cpu_time_in_nanos: 1898000, + memory_in_bytes: 24176, + }, + }, + ], + source: { + query: { + bool: { + must: [ + { + range: { + timestamp: { + from: 1726092595177, + to: 1726178995177, + include_lower: true, + include_upper: true, + boost: 1.0, + }, + }, + }, + ], + must_not: [ + { + match: { + indices: { + query: 'top_queries*', + operator: 'OR', + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: 'NONE', + auto_generate_synonyms_phrase_query: true, + boost: 1.0, + }, + }, + }, + ], + adjust_pure_negative: true, + boost: 1.0, + }, + }, + }, + query_hashcode: '80a17984b847133b8bf5e7d5dfbfa96c', + phase_latency_map: { + expand: 0, + query: 5, + fetch: 0, + }, + labels: { + 'X-Opaque-Id': 'ae6c1170-5f98-47f4-b7fc-09ebcf574b81', + }, + total_shards: 1, + search_type: 'query_then_fetch', + node_id: 'Q36D2z_NRGKim6EZZMgi6A', + indices: ['top_queries-2024.09.12'], + measurements: { + latency: { + number: 8, + count: 1, + aggregationType: 'NONE', + }, + cpu: { + number: 5510000, + count: 1, + aggregationType: 'NONE', + }, + }, + }, + ]; +}; + +export const MockGroups = (): SearchQueryRecord[] => { + return [ + { + timestamp: 1726178995210, + task_resource_usages: [ + { + action: 'indices:data/read/search[phase/query]', + taskId: 18809, + parentTaskId: 18808, + nodeId: 'Q36D2z_NRGKim6EZZMgi6A', + taskResourceUsage: { + cpu_time_in_nanos: 3612000, + memory_in_bytes: 123944, + }, + }, + { + action: 'indices:data/read/search', + taskId: 18808, + parentTaskId: -1, + nodeId: 'Q36D2z_NRGKim6EZZMgi6A', + taskResourceUsage: { + cpu_time_in_nanos: 1898000, + memory_in_bytes: 24176, + }, + }, + ], + source: { + query: { + bool: { + must: [ + { + range: { + timestamp: { + from: 1726092595177, + to: 1726178995177, + include_lower: true, + include_upper: true, + boost: 1.0, + }, + }, + }, + ], + must_not: [ + { + match: { + indices: { + query: 'top_queries*', + operator: 'OR', + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: 'NONE', + auto_generate_synonyms_phrase_query: true, + boost: 1.0, + }, + }, + }, + ], + adjust_pure_negative: true, + boost: 1.0, + }, + }, + }, + query_hashcode: '80a17984b847133b8bf5e7d5dfbfa96c', + phase_latency_map: { + expand: 0, + query: 5, + fetch: 0, + }, + labels: { + 'X-Opaque-Id': 'ae6c1170-5f98-47f4-b7fc-09ebcf574b81', + }, + total_shards: 1, + search_type: 'query_then_fetch', + node_id: 'Q36D2z_NRGKim6EZZMgi6A', + indices: ['top_queries-2024.09.12'], + measurements: { + latency: { + number: 8, + count: 5, + aggregationType: 'NONE', + }, + cpu: { + number: 5510000, + count: 5, + aggregationType: 'NONE', + }, + }, + }, + ]; +}; diff --git a/types/types.ts b/types/types.ts index 01f2f59..ca4ec71 100644 --- a/types/types.ts +++ b/types/types.ts @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { ISearchSource } from 'src/plugins/data/public'; - export interface SearchQueryRecord { timestamp: number; measurements: { @@ -14,12 +12,13 @@ export interface SearchQueryRecord { }; total_shards: number; node_id: string; - source: ISearchSource; + source: Record; labels: Record; search_type: string; indices: string[]; phase_latency_map: PhaseLatencyMap; task_resource_usages: Task[]; + query_hashcode: string; } export interface Measurement {