From 88aa03941fa0bfaecca14200f5142dcfc372f867 Mon Sep 17 00:00:00 2001 From: wujiasheng03 Date: Wed, 8 May 2024 11:56:23 +0800 Subject: [PATCH 1/5] add switch on lyrebird metrics --- lyrebird/event.py | 13 +++++++++---- tests/test_db.py | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index 3d148877..16fdf56a 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -46,6 +46,7 @@ def __init__(self): self.any_channel = [] self.broadcast_executor = ThreadPoolExecutor(thread_name_prefix='event-broadcast-') self.only_report_channel = application.config.get('event.only_report_channel', []) + self.lyrebird_metrics_report = application.config.get('event.lyrebird_metrics_report', True) def broadcast_handler(self, callback_fn, event, args, kwargs): """ @@ -78,11 +79,15 @@ def broadcast_handler(self, callback_fn, event, args, kwargs): except Exception: logger.error(f'Event callback function [{callback_fn.__name__}] error. {traceback.format_exc()}') finally: - event_end_time = time.time() - event_duration = (event_end_time - event_start_time) * 1000 # Report the operation of Event # Prevent loop reporting, and only time-consuming event(more than 1ms) are reported - if event.channel != 'lyrebird_metrics' and event_duration > 1: + if event.channel == 'lyrebird_metrics': + return + if not self.lyrebird_metrics_report: + return + event_end_time = time.time() + event_duration = (event_end_time - event_start_time) * 1000 + if event_duration > 1: trace_info = { 'channel': event.channel, 'callback_fn': callback_fn.__name__, @@ -166,7 +171,7 @@ def publish(self, channel, message, state=False, *args, **kwargs): } message['sender'] = sender_dict - if not (channel not in self.pubsub_channels and channel in self.only_report_channel): + if channel in self.pubsub_channels or channel not in self.only_report_channel: self.event_queue.put(Event(event_id, channel, message)) # TODO Remove state and raw data diff --git a/tests/test_db.py b/tests/test_db.py index 461a83d5..2bd6a913 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -17,7 +17,8 @@ def event_server(tmpdir): _conf = { 'ip': '127.0.0.1', - 'mock.port': 9090 + 'mock.port': 9090, + 'event.lyrebird_metrics_report': False } _personal_conf = personal_config_template application._cm = MockConfigManager(config=_conf, personal_config=_personal_conf, ROOT=Path(tmpdir), root=Path(tmpdir)) From 19dec1d7cdc911b0e571bbdea9d41f523c39c8ad Mon Sep 17 00:00:00 2001 From: wujiasheng03 Date: Wed, 8 May 2024 11:57:16 +0800 Subject: [PATCH 2/5] add config in unit test --- tests/test_common_api.py | 3 ++- tests/test_dm.py | 6 ++++-- tests/test_dm_api.py | 3 ++- tests/test_dm_format.py | 3 ++- tests/test_dm_label.py | 3 ++- tests/test_dm_temp.py | 1 + tests/test_event.py | 3 ++- tests/test_flow_api.py | 3 ++- 8 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_common_api.py b/tests/test_common_api.py index 32309515..b01728ef 100644 --- a/tests/test_common_api.py +++ b/tests/test_common_api.py @@ -9,7 +9,8 @@ 'mock.port': 9090, 'custom_key': 'custom_value', 'config.value.tojsonKey': ['custom.[a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}'], - 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"custom_key": "custom_value"} + 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"custom_key": "custom_value"}, + 'event.lyrebird_metrics_report': False } origin_data = '"keyA":"valueA","keyB":"{{config.get(\'custom.8df051be-4381-41b6-9252-120d9b558bf6\')}}","keyC":"valueC","keyD":"{{today}}"' diff --git a/tests/test_dm.py b/tests/test_dm.py index 478b8bcd..7908a2bd 100644 --- a/tests/test_dm.py +++ b/tests/test_dm.py @@ -358,7 +358,8 @@ def data_manager(root, tmpdir): 'ip': '127.0.0.1', 'mock.port': 9090, 'config.value.tojsonKey': ['custom.[a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}'], - 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"key": "value"} + 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"key": "value"}, + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) lyrebird.mock.context.application.socket_io = FakeSocketio() @@ -375,7 +376,8 @@ def test_load_from_path(root): 'ip': '127.0.0.1', 'mock.port': 9090, 'config.value.tojsonKey': ['custom.[a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}'], - 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"key": "value"} + 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"key": "value"}, + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) diff --git a/tests/test_dm_api.py b/tests/test_dm_api.py index 5e6c220c..c692517a 100644 --- a/tests/test_dm_api.py +++ b/tests/test_dm_api.py @@ -62,7 +62,8 @@ def client(root, tmpdir): _conf = { 'ip': '127.0.0.1', 'mock.port': 9090, - 'mock.data': 'data' + 'mock.data': 'data', + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) lyrebird.mock.context.application.socket_io = FakeSocketio() diff --git a/tests/test_dm_format.py b/tests/test_dm_format.py index 1d0fbb93..ecaf1866 100644 --- a/tests/test_dm_format.py +++ b/tests/test_dm_format.py @@ -12,7 +12,8 @@ def config(): 'mock.port': 9090, 'custom_key': 'custom_value', 'config.value.tojsonKey': ['custom.[a-z0-9]{8}(-[a-z0-9]{4}){3}-[a-z0-9]{12}'], - 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"custom_key": "custom_value"} + 'custom.8df051be-4381-41b6-9252-120d9b558bf6': {"custom_key": "custom_value"}, + 'event.lyrebird_metrics_report': False } application._cm = ConfigManager() application._cm.config = _conf diff --git a/tests/test_dm_label.py b/tests/test_dm_label.py index 09f8d9d8..69e5faa4 100644 --- a/tests/test_dm_label.py +++ b/tests/test_dm_label.py @@ -79,7 +79,8 @@ def root(tmpdir): def data_manager(root): _conf = { 'ip': '127.0.0.1', - 'mock.port': 9090 + 'mock.port': 9090, + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) _dm = dm.DataManager() diff --git a/tests/test_dm_temp.py b/tests/test_dm_temp.py index 59e83f5a..05a803c4 100644 --- a/tests/test_dm_temp.py +++ b/tests/test_dm_temp.py @@ -83,6 +83,7 @@ def data_manager(root, tmpdir): _conf = { 'ip': '127.0.0.1', 'mock.port': 9090, + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) lyrebird.mock.context.application.socket_io = FakeSocketio() diff --git a/tests/test_event.py b/tests/test_event.py index cab390e8..bdc2b3ad 100644 --- a/tests/test_event.py +++ b/tests/test_event.py @@ -29,7 +29,8 @@ def callback_tester(): def event_server(): _conf = { 'ip': '127.0.0.1', - 'mock.port': 9090 + 'mock.port': 9090, + 'event.lyrebird_metrics_report': False } application._cm = MockConfigManager(config=_conf) lyrebird.mock.context.application.socket_io = FakeSocketio() diff --git a/tests/test_flow_api.py b/tests/test_flow_api.py index 5d2528af..41d3a628 100644 --- a/tests/test_flow_api.py +++ b/tests/test_flow_api.py @@ -94,7 +94,8 @@ def test_func(flow): 'checker.workspace': '', 'checker.switch': { FILENAME: True - } + }, + 'event.lyrebird_metrics_report': False } From 421992b9b5746b14f5a17b0a49796d10d1e97eff Mon Sep 17 00:00:00 2001 From: wujiasheng03 Date: Wed, 8 May 2024 14:23:23 +0800 Subject: [PATCH 3/5] change switch locate --- lyrebird/event.py | 3 --- lyrebird/reporter.py | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index 16fdf56a..ef4fbc96 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -46,7 +46,6 @@ def __init__(self): self.any_channel = [] self.broadcast_executor = ThreadPoolExecutor(thread_name_prefix='event-broadcast-') self.only_report_channel = application.config.get('event.only_report_channel', []) - self.lyrebird_metrics_report = application.config.get('event.lyrebird_metrics_report', True) def broadcast_handler(self, callback_fn, event, args, kwargs): """ @@ -83,8 +82,6 @@ def broadcast_handler(self, callback_fn, event, args, kwargs): # Prevent loop reporting, and only time-consuming event(more than 1ms) are reported if event.channel == 'lyrebird_metrics': return - if not self.lyrebird_metrics_report: - return event_end_time = time.time() event_duration = (event_end_time - event_start_time) * 1000 if event_duration > 1: diff --git a/lyrebird/reporter.py b/lyrebird/reporter.py index 33af588a..6fe82a10 100644 --- a/lyrebird/reporter.py +++ b/lyrebird/reporter.py @@ -15,6 +15,7 @@ class Reporter: def __init__(self): self.scripts = [] workspace = application.config.get('reporter.workspace') + self.lyrebird_metrics_report = application.config.get('event.lyrebird_metrics_report', True) if not workspace: logger.debug(f'reporter.workspace not set.') else: @@ -50,6 +51,10 @@ def _read_reporter(self, workspace): self.scripts.append(_script_module.report) def report(self, data): + if not self.lyrebird_metrics_report: + if isinstance(data, dict) and data.get('channel') == 'lyrebird_metrics': + return + task_manager = application.server.get('task') def send_report(): From bacbb37be0125a6cb05d3c4ba58f1bcb93ab8103 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Wed, 8 May 2024 14:27:50 +0800 Subject: [PATCH 4/5] recover --- lyrebird/event.py | 3 +++ lyrebird/reporter.py | 5 ----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index ef4fbc96..16fdf56a 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -46,6 +46,7 @@ def __init__(self): self.any_channel = [] self.broadcast_executor = ThreadPoolExecutor(thread_name_prefix='event-broadcast-') self.only_report_channel = application.config.get('event.only_report_channel', []) + self.lyrebird_metrics_report = application.config.get('event.lyrebird_metrics_report', True) def broadcast_handler(self, callback_fn, event, args, kwargs): """ @@ -82,6 +83,8 @@ def broadcast_handler(self, callback_fn, event, args, kwargs): # Prevent loop reporting, and only time-consuming event(more than 1ms) are reported if event.channel == 'lyrebird_metrics': return + if not self.lyrebird_metrics_report: + return event_end_time = time.time() event_duration = (event_end_time - event_start_time) * 1000 if event_duration > 1: diff --git a/lyrebird/reporter.py b/lyrebird/reporter.py index 6fe82a10..33af588a 100644 --- a/lyrebird/reporter.py +++ b/lyrebird/reporter.py @@ -15,7 +15,6 @@ class Reporter: def __init__(self): self.scripts = [] workspace = application.config.get('reporter.workspace') - self.lyrebird_metrics_report = application.config.get('event.lyrebird_metrics_report', True) if not workspace: logger.debug(f'reporter.workspace not set.') else: @@ -51,10 +50,6 @@ def _read_reporter(self, workspace): self.scripts.append(_script_module.report) def report(self, data): - if not self.lyrebird_metrics_report: - if isinstance(data, dict) and data.get('channel') == 'lyrebird_metrics': - return - task_manager = application.server.get('task') def send_report(): From 44755f865daa363378cdf2dd46307e0a7bb71601 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Wed, 8 May 2024 15:08:23 +0800 Subject: [PATCH 5/5] version --- lyrebird/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lyrebird/version.py b/lyrebird/version.py index fdac5079..811c2daf 100644 --- a/lyrebird/version.py +++ b/lyrebird/version.py @@ -1,3 +1,3 @@ -IVERSION = (2, 26, 3) +IVERSION = (2, 26, 4) VERSION = ".".join(str(i) for i in IVERSION) LYREBIRD = "Lyrebird " + VERSION