From 11cd2701a8aad67c31ee2f2489e9c9b74582c8ab Mon Sep 17 00:00:00 2001 From: ai-jz <156989844+ai-jz@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:29:19 -0700 Subject: [PATCH] fix(openai): preserve zero cached tokens when cache reporting is enabled (#43005) Co-authored-by: ai-jz Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> --- .../srt/entrypoints/openai/serving_chat.py | 4 +- .../srt/entrypoints/openai/usage_processor.py | 13 +++--- .../entrypoints/openai/test_serving_chat.py | 11 +++-- .../openai/test_usage_processor.py | 45 +++++++++++++++++++ 4 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 test/registered/unit/entrypoints/openai/test_usage_processor.py diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 20328541e0..9034dfa192 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -810,8 +810,8 @@ class OpenAIServingChat(OpenAIServingBase): ) -> PromptTokensDetails | None: if not get_serving().enable_cache_report: return None - return UsageProcessor._details_if_cached( - content["meta_info"].get("cached_tokens", 0) + return PromptTokensDetails( + cached_tokens=content["meta_info"].get("cached_tokens", 0) ) def _reported_prompt_tokens(self, meta_info: dict[str, Any]) -> int: diff --git a/python/sglang/srt/entrypoints/openai/usage_processor.py b/python/sglang/srt/entrypoints/openai/usage_processor.py index 1c9ed62f3e..503496af54 100644 --- a/python/sglang/srt/entrypoints/openai/usage_processor.py +++ b/python/sglang/srt/entrypoints/openai/usage_processor.py @@ -9,11 +9,6 @@ from sglang.srt.entrypoints.openai.protocol import PromptTokensDetails, UsageInf class UsageProcessor: """Stateless helpers that turn raw token counts into a UsageInfo.""" - @staticmethod - def _details_if_cached(count: int) -> Optional[PromptTokensDetails]: - """Return PromptTokensDetails only when count > 0 (keeps JSON slim).""" - return PromptTokensDetails(cached_tokens=count) if count > 0 else None - @staticmethod def calculate_response_usage( responses: List[Dict[str, Any]], @@ -42,7 +37,7 @@ class UsageProcessor: responses[i]["meta_info"].get("cached_tokens", 0) for i in range(0, len(responses), n_choices) ) - cached_details = UsageProcessor._details_if_cached(cached_total) + cached_details = PromptTokensDetails(cached_tokens=cached_total) return UsageProcessor.calculate_token_usage( prompt_tokens=prompt_tokens, @@ -74,8 +69,10 @@ class UsageProcessor: total_completion_tokens = sum(completion_tokens.values()) cached_details = ( - UsageProcessor._details_if_cached( - sum(tok for idx, tok in cached_tokens.items() if idx % n_choices == 0) + PromptTokensDetails( + cached_tokens=sum( + tok for idx, tok in cached_tokens.items() if idx % n_choices == 0 + ) ) if enable_cache_report else None diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 27eb55fbe8..e471a93bf5 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -4126,9 +4126,14 @@ class ServingChatTestCase(CustomTestCase): enter_override( self, get_context().override_server_args(enable_cache_report=True) ) - usages = self._collect_continuous_usage(cached_tokens=6) - self.assertTrue(usages, "continuous_usage_stats attached no usage") - self.assertEqual(usages[0]["prompt_tokens_details"]["cached_tokens"], 6) + for cached_tokens in (0, 6): + with self.subTest(cached_tokens=cached_tokens): + usages = self._collect_continuous_usage(cached_tokens=cached_tokens) + self.assertTrue(usages, "continuous_usage_stats attached no usage") + self.assertEqual( + usages[0]["prompt_tokens_details"]["cached_tokens"], + cached_tokens, + ) def test_continuous_usage_omits_cached_tokens_when_report_disabled(self): """With cache reporting off, continuous_usage_stats must not leak cached tokens.""" diff --git a/test/registered/unit/entrypoints/openai/test_usage_processor.py b/test/registered/unit/entrypoints/openai/test_usage_processor.py new file mode 100644 index 0000000000..2d59d49793 --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_usage_processor.py @@ -0,0 +1,45 @@ +import unittest + +from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +class TestUsageProcessor(CustomTestCase): + def test_cache_details_follow_reporting_flag(self): + for enable_cache_report, cached_tokens in ( + (False, 7), + (True, 0), + (True, 7), + ): + with self.subTest( + enable_cache_report=enable_cache_report, + cached_tokens=cached_tokens, + ): + usages = ( + UsageProcessor.calculate_response_usage( + [{"meta_info": {"cached_tokens": cached_tokens}}], + enable_cache_report=enable_cache_report, + ), + UsageProcessor.calculate_streaming_usage( + prompt_tokens={0: 0}, + reasoning_tokens={0: 0}, + completion_tokens={0: 0}, + cached_tokens={0: cached_tokens}, + n_choices=1, + enable_cache_report=enable_cache_report, + ), + ) + for usage in usages: + details = usage.prompt_tokens_details + if enable_cache_report: + self.assertIsNotNone(details) + self.assertEqual(details.cached_tokens, cached_tokens) + else: + self.assertIsNone(details) + + +if __name__ == "__main__": + unittest.main()