[FIX] [benchmark] Fix flush_cache failure after warmup by waiting for server idle (#33527)

python/sglang/benchmark/serving.py

@@ -977,7 +977,13 @@ def flush_server_cache(base_url: str, backend: str) -> None:
     cache_endpoint = (
         "/reset_prefix_cache" if backend.startswith("vllm") else "/flush_cache"
     )
-    response = requests.post(base_url + cache_endpoint, headers=get_auth_headers())
+    # Pass timeout so the server waits for idle instead of failing immediately
+    params = {"timeout": 10.0} if not backend.startswith("vllm") else {}
+    response = requests.post(
+        base_url + cache_endpoint,
+        headers=get_auth_headers(),
+        params=params,
+    )
     response.raise_for_status()
 
 

test/registered/bench_fn/test_benchmark_datasets_api.py

@@ -112,13 +112,17 @@ class TestEmbeddingBenchmarkBackends(unittest.TestCase):
 
             flush_server_cache("http://127.0.0.1:8000", "vllm-embedding")
             post.assert_called_once_with(
-                "http://127.0.0.1:8000/reset_prefix_cache", headers={}
+                "http://127.0.0.1:8000/reset_prefix_cache",
+                headers={},
+                params={},
             )
             post.reset_mock()
 
             flush_server_cache("http://127.0.0.1:30000", "sglang-embedding")
             post.assert_called_once_with(
-                "http://127.0.0.1:30000/flush_cache", headers={}
+                "http://127.0.0.1:30000/flush_cache",
+                headers={},
+                params={"timeout": 10.0},
             )