fix(openai): preserve zero cached tokens when cache reporting is enabled (#43005)

Co-authored-by: ai-jz <ai-jz@users.noreply.github.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
ai-jz
2026-10-13 23:29:19 +08:00
committed by GitHub
co-authored by ai-jz Xinyuan Tong
parent 3f1e185f6e
commit 11cd2701a8
4 changed files with 60 additions and 13 deletions
@@ -810,8 +810,8 @@ class OpenAIServingChat(OpenAIServingBase):
) -> PromptTokensDetails | None:
if not get_serving().enable_cache_report:
return None
return UsageProcessor._details_if_cached(
content["meta_info"].get("cached_tokens", 0)
return PromptTokensDetails(
cached_tokens=content["meta_info"].get("cached_tokens", 0)
)
def _reported_prompt_tokens(self, meta_info: dict[str, Any]) -> int:
@@ -9,11 +9,6 @@ from sglang.srt.entrypoints.openai.protocol import PromptTokensDetails, UsageInf
class UsageProcessor:
"""Stateless helpers that turn raw token counts into a UsageInfo."""
@staticmethod
def _details_if_cached(count: int) -> Optional[PromptTokensDetails]:
"""Return PromptTokensDetails only when count > 0 (keeps JSON slim)."""
return PromptTokensDetails(cached_tokens=count) if count > 0 else None
@staticmethod
def calculate_response_usage(
responses: List[Dict[str, Any]],
@@ -42,7 +37,7 @@ class UsageProcessor:
responses[i]["meta_info"].get("cached_tokens", 0)
for i in range(0, len(responses), n_choices)
)
cached_details = UsageProcessor._details_if_cached(cached_total)
cached_details = PromptTokensDetails(cached_tokens=cached_total)
return UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens,
@@ -74,8 +69,10 @@ class UsageProcessor:
total_completion_tokens = sum(completion_tokens.values())
cached_details = (
UsageProcessor._details_if_cached(
sum(tok for idx, tok in cached_tokens.items() if idx % n_choices == 0)
PromptTokensDetails(
cached_tokens=sum(
tok for idx, tok in cached_tokens.items() if idx % n_choices == 0
)
)
if enable_cache_report
else None