From 11380be0ee121cdd84be60e8147934aec9fd0202 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Tue, 21 May 2024 18:58:10 +0800 Subject: [PATCH 1/7] fix multiprocess config setting always false --- Dockerfile | 6 +++--- lyrebird/manager.py | 2 -- lyrebird/version.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 47dd3a44..c5938367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,8 @@ COPY --from=nodebuilder /usr/src/lyrebird/client/ /usr/src/lyrebird/client/ RUN if [[ -n "$USE_MIRROR" ]] ; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories ; fi \ && apk update \ && apk add --no-cache build-base jpeg-dev zlib-dev libffi-dev openssl-dev redis \ - && if [[ -n "$USE_MIRROR" ]] ; then pip install --upgrade pip -i https://pypi.douban.com/simple && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis -i https://pypi.douban.com/simple ; else pip install --upgrade pip && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis ; fi \ - && if [[ -n "$USE_MIRROR" ]] ; then pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv -i https://pypi.douban.com/simple ; else pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv ; fi \ + && if [[ -n "$USE_MIRROR" ]] ; then pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis -i https://pypi.tuna.tsinghua.edu.cn/simple ; else pip install --upgrade pip && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis ; fi \ + && if [[ -n "$USE_MIRROR" ]] ; then pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv -i https://pypi.tuna.tsinghua.edu.cn/simple ; else pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv ; fi \ && rm -rf /usr/src \ && apk del --purge build-base jpeg-dev zlib-dev libffi-dev openssl-dev @@ -24,7 +24,7 @@ ARG USE_MIRROR ENV PYTHONUNBUFFERED 1 RUN if [[ -n "$USE_MIRROR" ]] ; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories ; fi \ && apk update \ - && apk add --no-cache jpeg zlib libffi openssl curl libstdc++ tzdata \ + && apk add --no-cache jpeg zlib libffi openssl curl libstdc++ tzdata redis\ && echo -e "#!/bin/sh\nexport PYTHONPATH=/usr/local/mitmenv\npython -c 'from mitmproxy.tools.main import mitmdump;mitmdump()' \$@" > /usr/local/bin/mitmdump \ && chmod a+x /usr/local/bin/mitmdump COPY --from=pybuilder /usr/local/lib/python3.8/site-packages /usr/local/lib/python3.8/site-packages diff --git a/lyrebird/manager.py b/lyrebird/manager.py index c1423e05..0c6873a9 100644 --- a/lyrebird/manager.py +++ b/lyrebird/manager.py @@ -108,8 +108,6 @@ def main(): custom_conf['redis_db'] = args.redis_db if args.enable_multiprocess and compat_redis_check(): custom_conf['enable_multiprocess'] = True - else: - custom_conf['enable_multiprocess'] = False application._cm = ConfigManager(conf_path_list=args.config, custom_conf=custom_conf) diff --git a/lyrebird/version.py b/lyrebird/version.py index d6a31cc9..ff181263 100644 --- a/lyrebird/version.py +++ b/lyrebird/version.py @@ -1,3 +1,3 @@ -IVERSION = (3, 0, 0) +IVERSION = (3, 0, 1) VERSION = ".".join(str(i) for i in IVERSION) LYREBIRD = "Lyrebird " + VERSION From 44596da2a6aaff731832792ded7919ceb847cfba Mon Sep 17 00:00:00 2001 From: noO0ob Date: Wed, 22 May 2024 15:47:54 +0800 Subject: [PATCH 2/7] fix process event kill --- lyrebird/event.py | 1 + lyrebird/mock/extra_mock_server/server.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index ac53ff88..8940407c 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -209,6 +209,7 @@ def __init__(self, no_start = False): self.broadcast_executor = ThreadPoolExecutor(thread_name_prefix='event-broadcast-') self.process_executor = CustomExecuteServer() self.publish_server = PublishServer() + application.server['event_process_executor'] = self.process_executor def broadcast_handler(self, func_info, event, args, kwargs, process_queue=None): """ diff --git a/lyrebird/mock/extra_mock_server/server.py b/lyrebird/mock/extra_mock_server/server.py index 65e1d19e..994e1f44 100644 --- a/lyrebird/mock/extra_mock_server/server.py +++ b/lyrebird/mock/extra_mock_server/server.py @@ -130,7 +130,7 @@ def init_app(config): async def _run_app(config): global semaphore - semaphore = asyncio.Semaphore(10) + semaphore = asyncio.Semaphore(20) app = init_app(config) port = config.get('extra.mock.port') From d4235a0bf7b3bacf27895641135cc148e32c5d5a Mon Sep 17 00:00:00 2001 From: noO0ob Date: Thu, 23 May 2024 10:33:21 +0800 Subject: [PATCH 3/7] fix bug --- lyrebird/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index 8940407c..62ebfeb0 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -209,7 +209,6 @@ def __init__(self, no_start = False): self.broadcast_executor = ThreadPoolExecutor(thread_name_prefix='event-broadcast-') self.process_executor = CustomExecuteServer() self.publish_server = PublishServer() - application.server['event_process_executor'] = self.process_executor def broadcast_handler(self, func_info, event, args, kwargs, process_queue=None): """ @@ -291,6 +290,7 @@ def async_start(self): self.process_executor.async_obj['publish_queue'] = self.publish_server.publish_msg_queue self.process_executor.async_obj['eventserver'] = EventServer self.process_executor.start() + application.server['event_process_executor'] = self.process_executor EventServer.async_starting = True def stop(self): From 9d66505d5833be4777b8e38b31a4eb49d7cf832a Mon Sep 17 00:00:00 2001 From: noO0ob Date: Thu, 23 May 2024 10:33:45 +0800 Subject: [PATCH 4/7] enable-multiprocess support set to False --- lyrebird/manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lyrebird/manager.py b/lyrebird/manager.py index 0c6873a9..9c73dfd1 100644 --- a/lyrebird/manager.py +++ b/lyrebird/manager.py @@ -79,7 +79,7 @@ def main(): parser.add_argument('--database', dest='database', help='Set a database path. Default is "~/.lyrebird/lyrebird.db"') parser.add_argument('--es', dest='extra_string', action='append', nargs=2, help='Set a custom config') parser.add_argument('--no-mitm', dest='no_mitm', action='store_true', help='Start without mitmproxy on 4272') - parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', action='store_true', help='change event based on multithread to multiprocess(reply on redis)') + parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', type=bool, help='change event based on multithread to multiprocess(reply on redis)') parser.add_argument('--redis-port', dest='redis_port', type=int, help='specifies the redis service port currently in use, defalut is 6379') parser.add_argument('--redis-ip', dest='redis_ip', help='specifies the redis service ip currently in use, defalut is localhost') parser.add_argument('--redis-db', dest='redis_db', help='specifies the redis service db currently in use, defalut is 0') @@ -106,8 +106,12 @@ def main(): custom_conf['redis_port'] = args.redis_port if args.redis_db: custom_conf['redis_db'] = args.redis_db - if args.enable_multiprocess and compat_redis_check(): + # --enable-multiprocess has the highest priority, + # When args.enable_multiprocess is None, it is controlled by config or defaults to False + if args.enable_multiprocess == True and compat_redis_check(): custom_conf['enable_multiprocess'] = True + elif args.enable_multiprocess == False: + custom_conf['enable_multiprocess'] = False application._cm = ConfigManager(conf_path_list=args.config, custom_conf=custom_conf) From 671be9552b385fae5320ac0a5e28d9c0f36c1060 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Thu, 23 May 2024 20:01:58 +0800 Subject: [PATCH 5/7] fix bug --- lyrebird/manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lyrebird/manager.py b/lyrebird/manager.py index 9c73dfd1..7ce71727 100644 --- a/lyrebird/manager.py +++ b/lyrebird/manager.py @@ -79,7 +79,7 @@ def main(): parser.add_argument('--database', dest='database', help='Set a database path. Default is "~/.lyrebird/lyrebird.db"') parser.add_argument('--es', dest='extra_string', action='append', nargs=2, help='Set a custom config') parser.add_argument('--no-mitm', dest='no_mitm', action='store_true', help='Start without mitmproxy on 4272') - parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', type=bool, help='change event based on multithread to multiprocess(reply on redis)') + parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', help='change event based on multithread to multiprocess(reply on redis)') parser.add_argument('--redis-port', dest='redis_port', type=int, help='specifies the redis service port currently in use, defalut is 6379') parser.add_argument('--redis-ip', dest='redis_ip', help='specifies the redis service ip currently in use, defalut is localhost') parser.add_argument('--redis-db', dest='redis_db', help='specifies the redis service db currently in use, defalut is 0') @@ -108,9 +108,10 @@ def main(): custom_conf['redis_db'] = args.redis_db # --enable-multiprocess has the highest priority, # When args.enable_multiprocess is None, it is controlled by config or defaults to False - if args.enable_multiprocess == True and compat_redis_check(): + enable_multiprocess = args.enable_multiprocess.lower() if isinstance(args.enable_multiprocess, str) else None + if enable_multiprocess == 'true' and compat_redis_check(): custom_conf['enable_multiprocess'] = True - elif args.enable_multiprocess == False: + elif enable_multiprocess == 'false': custom_conf['enable_multiprocess'] = False application._cm = ConfigManager(conf_path_list=args.config, custom_conf=custom_conf) From c3f500c269bf959607ef6c4738e5e07ec974e909 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Fri, 24 May 2024 10:09:38 +0800 Subject: [PATCH 6/7] add event multiprocess report --- lyrebird/event.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lyrebird/event.py b/lyrebird/event.py index 62ebfeb0..d6c220fc 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -292,6 +292,7 @@ def async_start(self): self.process_executor.start() application.server['event_process_executor'] = self.process_executor EventServer.async_starting = True + self.publish('system', {'action': 'event.multiprocess', 'module': 'EventServer', 'status': 'READY'}) def stop(self): self.publish('system', {'name': 'event.stop'}) From 07d85cf1010f3ed6c9d35d02facdc72fa9623893 Mon Sep 17 00:00:00 2001 From: noO0ob Date: Fri, 24 May 2024 10:47:27 +0800 Subject: [PATCH 7/7] fix bug --- lyrebird/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lyrebird/event.py b/lyrebird/event.py index d6c220fc..de4ffc03 100644 --- a/lyrebird/event.py +++ b/lyrebird/event.py @@ -292,7 +292,7 @@ def async_start(self): self.process_executor.start() application.server['event_process_executor'] = self.process_executor EventServer.async_starting = True - self.publish('system', {'action': 'event.multiprocess', 'module': 'EventServer', 'status': 'READY'}) + self.publish('system', {'system': {'action': 'event.multiprocess', 'module': 'event_server', 'status': 'READY'}}) def stop(self): self.publish('system', {'name': 'event.stop'})