fix(openai): preserve zero cached tokens when cache reporting is enabled (#43005)
Co-authored-by: ai-jz <ai-jz@users.noreply.github.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
co-authored by
ai-jz
Xinyuan Tong
parent
3f1e185f6e
commit
11cd2701a8
@@ -810,8 +810,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
) -> PromptTokensDetails | None:
|
||||
if not get_serving().enable_cache_report:
|
||||
return None
|
||||
return UsageProcessor._details_if_cached(
|
||||
content["meta_info"].get("cached_tokens", 0)
|
||||
return PromptTokensDetails(
|
||||
cached_tokens=content["meta_info"].get("cached_tokens", 0)
|
||||
)
|
||||
|
||||
def _reported_prompt_tokens(self, meta_info: dict[str, Any]) -> int:
|
||||
|
||||
@@ -9,11 +9,6 @@ from sglang.srt.entrypoints.openai.protocol import PromptTokensDetails, UsageInf
|
||||
class UsageProcessor:
|
||||
"""Stateless helpers that turn raw token counts into a UsageInfo."""
|
||||
|
||||
@staticmethod
|
||||
def _details_if_cached(count: int) -> Optional[PromptTokensDetails]:
|
||||
"""Return PromptTokensDetails only when count > 0 (keeps JSON slim)."""
|
||||
return PromptTokensDetails(cached_tokens=count) if count > 0 else None
|
||||
|
||||
@staticmethod
|
||||
def calculate_response_usage(
|
||||
responses: List[Dict[str, Any]],
|
||||
@@ -42,7 +37,7 @@ class UsageProcessor:
|
||||
responses[i]["meta_info"].get("cached_tokens", 0)
|
||||
for i in range(0, len(responses), n_choices)
|
||||
)
|
||||
cached_details = UsageProcessor._details_if_cached(cached_total)
|
||||
cached_details = PromptTokensDetails(cached_tokens=cached_total)
|
||||
|
||||
return UsageProcessor.calculate_token_usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
@@ -74,8 +69,10 @@ class UsageProcessor:
|
||||
total_completion_tokens = sum(completion_tokens.values())
|
||||
|
||||
cached_details = (
|
||||
UsageProcessor._details_if_cached(
|
||||
sum(tok for idx, tok in cached_tokens.items() if idx % n_choices == 0)
|
||||
PromptTokensDetails(
|
||||
cached_tokens=sum(
|
||||
tok for idx, tok in cached_tokens.items() if idx % n_choices == 0
|
||||
)
|
||||
)
|
||||
if enable_cache_report
|
||||
else None
|
||||
|
||||
Reference in New Issue
Block a user