From 877a293d6d2d9d7266467f7253aea8ec7b970203 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Mon, 21 Sep 2026 21:31:21 -0700 Subject: [PATCH] [Benchmark] Optionally clear HiCache storage between cases (#40659) Co-authored-by: metamergebot <324680979+metamergebot@users.noreply.github.com> Co-authored-by: Pengchao Wang --- python/sglang/benchmark/one_batch_server.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/sglang/benchmark/one_batch_server.py b/python/sglang/benchmark/one_batch_server.py index 10d9755e1..4084e0f4f 100644 --- a/python/sglang/benchmark/one_batch_server.py +++ b/python/sglang/benchmark/one_batch_server.py @@ -149,6 +149,7 @@ class BenchArgs: cache_hit_rate: float = 0.0 backend: str = "sglang" fake_prefill: bool = False + flush_hicache_storage: bool = False server_args_for_metrics: Optional[List[str]] = None lora_name: Optional[List[str]] = None lora_request_distribution: str = "uniform" @@ -348,6 +349,13 @@ class BenchArgs: "Use with a decode server running --disaggregation-transfer-backend fake " "to benchmark pure decode performance without a real prefill node.", ) + parser.add_argument( + "--flush-hicache-storage", + action="store_true", + default=BenchArgs.flush_hicache_storage, + help="Also clear the hierarchical cache's storage tier before each case; " + "/flush_cache resets the radix tree and the host tier only.", + ) parser.add_argument( "--server-args-for-metrics", type=str, @@ -624,12 +632,17 @@ def run_one_case( lora_zipf_alpha: float = BenchArgs.lora_zipf_alpha, fixed_prompt_file: str = "", apply_chat_template: bool = False, + flush_hicache_storage: bool = False, ): if backend == "vllm": # You need to have export VLLM_SERVER_DEV_MODE=1 in your environment to use this endpoint. _flush_cache_with_retry(url, "/reset_prefix_cache") else: _flush_cache_with_retry(url, "/flush_cache") + # /flush_cache resets the radix tree and the host tier; a storage tier + # persists across it and would serve the same prompts on the next case. + if flush_hicache_storage: + _flush_cache_with_retry(url, "/hicache/storage-backend/clear") if fixed_prompt_file: tok_inner = getattr(tokenizer, "tokenizer", tokenizer) @@ -1360,6 +1373,7 @@ def run_benchmark_internal( backend=bench_args.backend, model_name=model_name, fake_prefill=bench_args.fake_prefill, + flush_hicache_storage=bench_args.flush_hicache_storage, lora_name=bench_args.lora_name, lora_request_distribution=bench_args.lora_request_distribution, lora_zipf_alpha=bench_args.lora_zipf_alpha, @@ -1406,6 +1420,7 @@ def run_benchmark_internal( backend=bench_args.backend, model_name=model_name, fake_prefill=bench_args.fake_prefill, + flush_hicache_storage=bench_args.flush_hicache_storage, lora_name=bench_args.lora_name, lora_request_distribution=bench_args.lora_request_distribution, lora_zipf_alpha=bench_args.lora_zipf_alpha, @@ -1463,6 +1478,7 @@ def run_benchmark_internal( backend=bench_args.backend, model_name=model_name, fake_prefill=bench_args.fake_prefill, + flush_hicache_storage=bench_args.flush_hicache_storage, lora_name=bench_args.lora_name, lora_request_distribution=bench_args.lora_request_distribution, lora_zipf_alpha=bench_args.lora_zipf_alpha,