minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Fix cached tokens usage bug (#12814)

Browse Source 
Co-authored-by: FrankMinions <liuchen@shinemo.com>
This commit is contained in:
Frank Minors
2025-11-12 01:02:43 +08:00
committed by GitHub
co-authored by FrankMinions
parent 63728b1154
commit 7f5055ed5e
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/usage_processor.py
+5 -2
View File
@@ -30,7 +30,8 @@ class UsageProcessor:
cached_details = None cached_details = None
if enable_cache_report: if enable_cache_report:
cached_total = sum( cached_total = sum(
r["meta_info"].get("cached_tokens", 0) for r in responses responses[i]["meta_info"].get("cached_tokens", 0)
for i in range(0, len(responses), n_choices)
) )
cached_details = UsageProcessor._details_if_cached(cached_total) cached_details = UsageProcessor._details_if_cached(cached_total)
@@ -55,7 +56,9 @@ class UsageProcessor:
total_completion_tokens = sum(completion_tokens.values()) total_completion_tokens = sum(completion_tokens.values())
cached_details = ( cached_details = (
UsageProcessor._details_if_cached(sum(cached_tokens.values())) UsageProcessor._details_if_cached(
sum(tok for idx, tok in cached_tokens.items() if idx % n_choices == 0)
)
if enable_cache_report if enable_cache_report
else None else None
) )