minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Tiny fix bench serving GSP mode cache file strategy (#15587)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-22 11:27:55 +08:00
committed by GitHub
parent 468931b572
commit f4100732b8
1 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/bench_serving.py
+7 -5
View File
@@ -1651,9 +1651,10 @@ def sample_generated_shared_prefix_requests(
) -> List[DatasetRow]: ) -> List[DatasetRow]:
"""Generate benchmark requests with shared system prompts using random tokens and caching.""" """Generate benchmark requests with shared system prompts using random tokens and caching."""
cache_path = get_gen_prefix_cache_path(args, tokenizer) cache_path = get_gen_prefix_cache_path(args, tokenizer)
should_cache = range_ratio == 1
# Try to load from cache first # Try to load from cache first
if cache_path.exists() and range_ratio == 1: if cache_path.exists() and should_cache:
print(f"\nLoading cached generated input data from {cache_path}") print(f"\nLoading cached generated input data from {cache_path}")
with open(cache_path, "rb") as f: with open(cache_path, "rb") as f:
return pickle.load(f) return pickle.load(f)
@@ -1740,10 +1741,11 @@ def sample_generated_shared_prefix_requests(
) )
# Save to cache # Save to cache
cache_path.parent.mkdir(parents=True, exist_ok=True) if should_cache:
print(f"Caching generated input data to {cache_path}") cache_path.parent.mkdir(parents=True, exist_ok=True)
with open(cache_path, "wb") as f: print(f"Caching generated input data to {cache_path}")
pickle.dump(input_requests, f) with open(cache_path, "wb") as f:
pickle.dump(input_requests, f)
return input_requests return input_requests