stable api_server benchmark result by a non-zero await (#885)

* stable api_server benchmark result by a non-zero await * move to beginning of the loop
InternLM · Dec 25, 2023 · e84e99a · e84e99a
1 parent 174474a
commit e84e99a
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/lmdeploy/turbomind/turbomind.py b/lmdeploy/turbomind/turbomind.py
@@ -650,7 +650,8 @@ async def async_stream_infer(self,
             # Thanks for https://github.com/frankxyy and his issue
             # https://github.com/InternLM/lmdeploy/issues/832
             while self.que.qsize() == 0:
-                await asyncio.sleep(0)
+                await asyncio.sleep(0.002)  # sleep(0) makes server unstable
+
             while self.que.qsize() > 1:
                 self.que.get()