From e84e99ad5bcc4412c559fda48ab2279d0cc5a060 Mon Sep 17 00:00:00 2001
From: AllentDan <41138331+AllentDan@users.noreply.github.com>
Date: Mon, 25 Dec 2023 17:10:58 +0800
Subject: [PATCH] stable api_server benchmark result by a non-zero await (#885)

* stable api_server benchmark result by a non-zero await

* move to beginning of the loop
---
 lmdeploy/turbomind/turbomind.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lmdeploy/turbomind/turbomind.py b/lmdeploy/turbomind/turbomind.py
index 7d6077cd47..b1ad357462 100644
--- a/lmdeploy/turbomind/turbomind.py
+++ b/lmdeploy/turbomind/turbomind.py
@@ -650,7 +650,8 @@ async def async_stream_infer(self,
             # Thanks for https://github.com/frankxyy and his issue
             # https://github.com/InternLM/lmdeploy/issues/832
             while self.que.qsize() == 0:
-                await asyncio.sleep(0)
+                await asyncio.sleep(0.002)  # sleep(0) makes server unstable
+
             while self.que.qsize() > 1:
                 self.que.get()