Simplify flush_cache: reject concurrent requests, remove client-side retry (#21490)

python/sglang/srt/entrypoints/http_server.py

@@ -736,9 +736,15 @@ async def classify_request(obj: EmbeddingReqInput, request: Request):
 async def flush_cache(timeout: float = Query(0.0, ge=0.0)):
     """Flush the radix cache."""
     ret = await _global_state.tokenizer_manager.flush_cache(timeout_s=timeout)
+    if ret.success:
+        content = (
+            "Cache flushed.\nPlease check backend logs for more details. "
+            "(When there are running or waiting requests, the operation will not be performed.)\n"
+        )
+    else:
+        content = ret.message or "Flush cache failed.\n"
     return Response(
-        content="Cache flushed.\nPlease check backend logs for more details. "
-        "(When there are running or waiting requests, the operation will not be performed.)\n",
+        content=content,
         status_code=200 if ret.success else HTTPStatus.BAD_REQUEST,
     )
 

python/sglang/srt/managers/io_struct.py

@@ -1172,6 +1172,7 @@ class FlushCacheReqInput(BaseReq):
 @dataclass
 class FlushCacheReqOutput(BaseReq):
     success: bool
+    message: str = ""
 
 
 @dataclass

python/sglang/srt/managers/scheduler.py

@@ -834,7 +834,7 @@ class Scheduler(
         self.last_batch: Optional[ScheduleBatch] = None
         self.forward_ct = 0
         self.return_health_check_ipcs: Deque[Optional[str]] = deque()
-        self._pending_flush: Deque[Tuple[FlushCacheReqInput, float]] = deque()
+        self._pending_flush: Optional[Tuple[FlushCacheReqInput, float]] = None
         self.num_retracted_reqs: int = 0
         self.num_paused_reqs: int = 0
         self.session_controller = SessionController(self.tree_cache)
@@ -2785,38 +2785,40 @@ class Scheduler(
             )
 
     def _check_pending_flush(self):
-        if not self._pending_flush:
+        if self._pending_flush is None:
             return
 
+        pending_req, deadline = self._pending_flush
+
         if self.is_fully_idle():
             success = self.flush_cache()
-            while self._pending_flush:
-                pending_req, _ = self._pending_flush.popleft()
-                self.send_to_tokenizer.send_output(
-                    FlushCacheReqOutput(success=success), pending_req
-                )
+            self._pending_flush = None
+            self.send_to_tokenizer.send_output(
+                FlushCacheReqOutput(success=success), pending_req
+            )
             return
 
-        self._expire_timed_out_pending_flushes(time.monotonic())
-
-    def _expire_timed_out_pending_flushes(self, now: float):
-        remaining: Deque[Tuple[FlushCacheReqInput, float]] = deque()
-        while self._pending_flush:
-            pending_req, deadline = self._pending_flush.popleft()
-            if now >= deadline:
-                logging.warning(
-                    "Deferred flush_cache timed out while waiting for idle state."
-                )
-                self.send_to_tokenizer.send_output(
-                    FlushCacheReqOutput(success=False), pending_req
-                )
-            else:
-                remaining.append((pending_req, deadline))
-        self._pending_flush = remaining
+        if time.monotonic() >= deadline:
+            logging.warning(
+                "Deferred flush_cache timed out while waiting for idle state."
+            )
+            self._pending_flush = None
+            self.send_to_tokenizer.send_output(
+                FlushCacheReqOutput(
+                    success=False, message="Timed out waiting for idle state."
+                ),
+                pending_req,
+            )
 
     def flush_cache_wrapped(
         self, recv_req: FlushCacheReqInput
     ) -> Optional[FlushCacheReqOutput]:
+        if self._pending_flush is not None:
+            return FlushCacheReqOutput(
+                success=False,
+                message="Another flush_cache is already in progress.",
+            )
+
         timeout_s = float(recv_req.timeout_s or 0.0)
         if timeout_s <= 0.0:
             return FlushCacheReqOutput(success=self.flush_cache())
@@ -2824,7 +2826,7 @@ class Scheduler(
         if self.is_fully_idle():
             return FlushCacheReqOutput(success=self.flush_cache())
 
-        self._pending_flush.append((recv_req, time.monotonic() + timeout_s))
+        self._pending_flush = (recv_req, time.monotonic() + timeout_s)
         return None
 
     def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput):

python/sglang/test/test_utils.py

@@ -168,31 +168,6 @@ def download_image_with_retry(image_url: str, max_retries: int = 3) -> Image.Ima
             time.sleep(2**i)
 
 
-def flush_cache_with_retry(
-    base_url: str,
-    timeout: float = 30.0,
-    poll_interval: float = 0.5,
-) -> bool:
-    """Flush device cache, polling until success or timeout.
-
-    flush_cache only succeeds when the scheduler is fully idle, but
-    HiCache async ops (write-through, backup) may still be in-flight
-    after a request completes.  We poll with a short interval so idle
-    is detected quickly, while the generous timeout accommodates slow
-    CI environments.
-    """
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        try:
-            response = requests.post(f"{base_url}/flush_cache", timeout=10)
-            if response.status_code == 200:
-                return True
-        except requests.RequestException:
-            pass
-        time.sleep(poll_interval)
-    return False
-
-
 def is_in_ci():
     """Return whether it is in CI runner."""
     return get_bool_env_var("SGLANG_IS_IN_CI")