minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Anthropic] Fix missing cache_read_input_tokens in streaming responses (#29703)

Browse Source
This commit is contained in:
Yiqi Yang
2026-07-03 12:46:45 +08:00
committed by GitHub
parent 860244d4b4
commit f011d8c2e5
2 changed files with 72 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_chat.py
+15
View File
@@ -38,6 +38,7 @@ from sglang.srt.entrypoints.openai.protocol import (
FunctionResponse,
LogProbs,
MessageProcessingResult,
PromptTokensDetails,
ResponseParserProtocol,
SglExt,
ToolCall,
@@ -336,6 +337,15 @@ class OpenAIServingChat(OpenAIServingBase):
"""Post-process reasoning and tool_calls before building response."""
return reasoning_text, tool_calls
def _continuous_usage_cached_details(
self, content: Dict[str, Any]
) -> Optional[PromptTokensDetails]:
if not self.tokenizer_manager.server_args.enable_cache_report:
return None
return UsageProcessor._details_if_cached(
content["meta_info"].get("cached_tokens", 0)
)
async def _generate_stream_content(
self,
content: Dict[str, Any],
@@ -377,6 +387,7 @@ class OpenAIServingChat(OpenAIServingBase):
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
@@ -422,6 +433,7 @@ class OpenAIServingChat(OpenAIServingBase):
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
@@ -449,6 +461,7 @@ class OpenAIServingChat(OpenAIServingBase):
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
@@ -1898,6 +1911,7 @@ class OpenAIServingChat(OpenAIServingBase):
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
cached_tokens=self._continuous_usage_cached_details(content),
)
yield f"data: {chunk.model_dump_json()}\n\n"
@@ -1950,6 +1964,7 @@ class OpenAIServingChat(OpenAIServingBase):
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
cached_tokens=self._continuous_usage_cached_details(content),
)
yield f"data: {chunk.model_dump_json()}\n\n"