From 5f2861bf599767c1d174b434333bd4b97ee57b4a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 14 Oct 2026 16:41:34 -0700 Subject: [PATCH] [Perf] session: Cache multimodal prompt token counts across turns (#44112) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: joshuadeng --- python/sglang/srt/disaggregation/decode.py | 1 + python/sglang/srt/disaggregation/utils.py | 5 +- python/sglang/srt/managers/schedule_batch.py | 28 +++++++++ .../scheduler_components/output_streamer.py | 9 +-- .../sglang/srt/session/session_controller.py | 3 + .../test_disaggregation_wire.py | 1 + .../unit/managers/test_mm_token_counts.py | 57 +++++++++++++++++++ .../test_output_streamer_customized_info.py | 3 + .../managers/test_output_streamer_logprobs.py | 3 + .../test_session_token_share_unit.py | 15 +++++ 10 files changed, 113 insertions(+), 12 deletions(-) create mode 100644 test/registered/unit/managers/test_mm_token_counts.py diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index d6621f7995..6595f678ad 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -2629,6 +2629,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): decode_req.req.mm_image_tokens = cached_tokens[4].item() decode_req.req.mm_audio_tokens = cached_tokens[5].item() decode_req.req.mm_video_tokens = cached_tokens[6].item() + decode_req.req.mm_token_counts_ready = True if not self.spec_algorithm.is_none(): decode_req.req.output_topk_p = output_topk_p decode_req.req.output_topk_index = output_topk_index diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 4869439a7c..b3090330d6 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -549,10 +549,7 @@ class MetadataBuffers: # Compute multimodal prompt token counts on the prefill node so decode # can report them in usage. - if req.multimodal_inputs: - image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts() - else: - image_t = audio_t = video_t = 0 + image_t, audio_t, video_t = req.get_mm_token_counts() self.cached_tokens[req.metadata_buffer_index][4] = image_t self.cached_tokens[req.metadata_buffer_index][5] = audio_t self.cached_tokens[req.metadata_buffer_index][6] = video_t diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index e100386662..8beec775a2 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1180,6 +1180,7 @@ class Req(ReqDllmMixin): self.mm_image_tokens: int = 0 self.mm_audio_tokens: int = 0 self.mm_video_tokens: int = 0 + self.mm_token_counts_ready: bool = False # Prefix info # Tokens [0, prefix_len) are already in the row, or, before allocation, @@ -1427,6 +1428,26 @@ class Req(ReqDllmMixin): # Snapshot of the scheduler prefill-token counter taken at waiting_queue entry; used by HRRN aging. self.arrival_processed_tokens: int = 0 + def get_mm_token_counts(self) -> Tuple[int, int, int]: + """Return per-modality prompt counts, scanning media at most once.""" + if not self.mm_token_counts_ready and self.multimodal_inputs is not None: + ( + self.mm_image_tokens, + self.mm_audio_tokens, + self.mm_video_tokens, + ) = self.multimodal_inputs.compute_mm_token_counts() + self.mm_token_counts_ready = True + return self.mm_image_tokens, self.mm_audio_tokens, self.mm_video_tokens + + def inherit_mm_token_counts(self, other: Req) -> None: + """Reuse counts from the request whose media history is shared.""" + ( + self.mm_image_tokens, + self.mm_audio_tokens, + self.mm_video_tokens, + ) = other.get_mm_token_counts() + self.mm_token_counts_ready = True + def advance_cache_request_handle(self) -> None: self.cache_request_handle = dataclasses.replace( self.cache_request_handle, @@ -1519,6 +1540,13 @@ class Req(ReqDllmMixin): self.spec_cap_lens_histogram[cap_len] += 1 def extend_image_inputs(self, image_inputs): + if self.mm_token_counts_ready: + image_tokens, audio_tokens, video_tokens = ( + image_inputs.compute_mm_token_counts() + ) + self.mm_image_tokens += image_tokens + self.mm_audio_tokens += audio_tokens + self.mm_video_tokens += video_tokens if self.session is not None: self._extend_session_image_inputs(image_inputs) elif self.multimodal_inputs is None: diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index d2dd5a8253..fedd34247d 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -513,14 +513,7 @@ class _GenerationStreamAccumulator: # already computed these and transferred them via the metadata buffer # (req.mm_*), so prefer the pre-stored values; otherwise compute them # from the request's multimodal items. - if req.mm_image_tokens or req.mm_audio_tokens or req.mm_video_tokens: - image_t = req.mm_image_tokens - audio_t = req.mm_audio_tokens - video_t = req.mm_video_tokens - elif req.multimodal_inputs: - image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts() - else: - image_t = audio_t = video_t = 0 + image_t, audio_t, video_t = req.get_mm_token_counts() self.image_tokens.append(image_t) self.audio_tokens.append(audio_t) self.video_tokens.append(video_t) diff --git a/python/sglang/srt/session/session_controller.py b/python/sglang/srt/session/session_controller.py index 8ff2044bc0..f291cdc26e 100644 --- a/python/sglang/srt/session/session_controller.py +++ b/python/sglang/srt/session/session_controller.py @@ -332,6 +332,9 @@ class Session: ) if last_req is not None: new_req.multimodal_inputs = last_req.multimodal_inputs + # An offset can truncate inherited media. + if not abort and not session_params.offset: + new_req.inherit_mm_token_counts(last_req) new_req.tokenizer = tokenizer if carry_fill is not None: new_req.full_untruncated_fill_ids = carry_fill diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 33702a5cd6..2b54fbe60b 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -1748,6 +1748,7 @@ class TestEagleDsaSeedTransfer(CustomTestCase): cached_tokens_device=0, cached_tokens_host=0, cached_tokens_storage=0, + get_mm_token_counts=lambda: (0, 0, 0), multimodal_inputs=None, return_logprob=False, return_sampling_mask=sampling_mask is not None, diff --git a/test/registered/unit/managers/test_mm_token_counts.py b/test/registered/unit/managers/test_mm_token_counts.py new file mode 100644 index 0000000000..91255f0cec --- /dev/null +++ b/test/registered/unit/managers/test_mm_token_counts.py @@ -0,0 +1,57 @@ +import unittest +from array import array +from unittest.mock import MagicMock + +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, + Req, +) +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _item(modality, *offsets): + return MultimodalDataItem(modality=modality, offsets=list(offsets) or None) + + +def _req(rid): + return Req(rid, "", array("q"), SamplingParams(max_new_tokens=1)) + + +class TestMultimodalTokenCounts(unittest.TestCase): + def test_request_memoizes_empty_counts(self): + """All-zero counts must be cached too; a zero result is not a cache miss.""" + req = _req("empty") + req.multimodal_inputs = mm = MagicMock() + mm.compute_mm_token_counts.return_value = (0, 0, 0) + + self.assertEqual(req.get_mm_token_counts(), (0, 0, 0)) + self.assertEqual(req.get_mm_token_counts(), (0, 0, 0)) + mm.compute_mm_token_counts.assert_called_once_with() + + def test_session_request_inherits_and_extends_counts(self): + """A session turn reuses the prior turn's counts and adds only new media.""" + parent = _req("parent") + audio = _item(Modality.AUDIO, (0, 3)) + parent.multimodal_inputs = history = MultimodalInputs(mm_items=[audio]) + + child = _req("child") + child.session = object() + child.origin_input_ids = array("q", range(8)) + child.multimodal_inputs = history + child.inherit_mm_token_counts(parent) + child.extend_image_inputs( + MultimodalInputs(mm_items=[_item(Modality.IMAGE, (4, 7))]) + ) + + self.assertEqual(parent.get_mm_token_counts(), (0, 4, 0)) + self.assertEqual(child.get_mm_token_counts(), (4, 4, 0)) + self.assertIsNot(child.multimodal_inputs, history) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_output_streamer_customized_info.py b/test/registered/unit/managers/test_output_streamer_customized_info.py index 483466e1f9..d00e70f26a 100644 --- a/test/registered/unit/managers/test_output_streamer_customized_info.py +++ b/test/registered/unit/managers/test_output_streamer_customized_info.py @@ -91,6 +91,9 @@ class _FakeReq: def finished(self): return self._finished + def get_mm_token_counts(self): + return self.mm_image_tokens, self.mm_audio_tokens, self.mm_video_tokens + def init_incremental_detokenize(self): return self.output_ids_through_stop, 0 diff --git a/test/registered/unit/managers/test_output_streamer_logprobs.py b/test/registered/unit/managers/test_output_streamer_logprobs.py index d50736b0fb..e85a7d5f19 100644 --- a/test/registered/unit/managers/test_output_streamer_logprobs.py +++ b/test/registered/unit/managers/test_output_streamer_logprobs.py @@ -59,6 +59,9 @@ class _FakeReq: def finished(self): return False + def get_mm_token_counts(self): + return self.mm_image_tokens, self.mm_audio_tokens, self.mm_video_tokens + def init_incremental_detokenize(self): return self.output_ids_through_stop, 0 diff --git a/test/registered/unit/mem_cache/test_session_token_share_unit.py b/test/registered/unit/mem_cache/test_session_token_share_unit.py index 7b70670724..6a69a7bda6 100644 --- a/test/registered/unit/mem_cache/test_session_token_share_unit.py +++ b/test/registered/unit/mem_cache/test_session_token_share_unit.py @@ -105,6 +105,21 @@ class TestSessionTokenShare(CustomTestCase): self.assertEqual(list(r3.origin_input_ids), in1 + out1 + in2 + out2 + [9]) self.assertEqual(list(r3.full_untruncated_fill_ids), list(r3.origin_input_ids)) + def test_multimodal_counts_follow_committed_history(self): + """An aborted turn's count edits must not leak into the next turn.""" + r1 = self._create("r1", [1]) + r1.mm_image_tokens, r1.mm_audio_tokens, r1.mm_video_tokens = (2, 3, 4) + r1.mm_token_counts_ready = True + self._decode_and_finish(r1, []) + + r2 = self._create("r2", [2]) + self.assertEqual(r2.get_mm_token_counts(), (2, 3, 4)) + r2.mm_image_tokens = 5 + self.session.abort_req() + + r3 = self._create("r3", [3]) + self.assertEqual(r3.get_mm_token_counts(), (2, 3, 4)) + def test_mid_turn_abort_then_continue(self): in1, out1 = list(range(200, 210)), [1, 2, 3] r1 = self._create("r1", in1)