[Perf] session: Cache multimodal prompt token counts across turns (#44112)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-14 16:41:34 -07:00
committed by GitHub
co-authored by Hanming Lu joshuadeng
parent 9732188a58
commit 5f2861bf59
10 changed files with 113 additions and 12 deletions
@@ -2629,6 +2629,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
decode_req.req.mm_image_tokens = cached_tokens[4].item()
decode_req.req.mm_audio_tokens = cached_tokens[5].item()
decode_req.req.mm_video_tokens = cached_tokens[6].item()
decode_req.req.mm_token_counts_ready = True
if not self.spec_algorithm.is_none():
decode_req.req.output_topk_p = output_topk_p
decode_req.req.output_topk_index = output_topk_index
+1 -4
View File
@@ -549,10 +549,7 @@ class MetadataBuffers:
# Compute multimodal prompt token counts on the prefill node so decode
# can report them in usage.
if req.multimodal_inputs:
image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts()
else:
image_t = audio_t = video_t = 0
image_t, audio_t, video_t = req.get_mm_token_counts()
self.cached_tokens[req.metadata_buffer_index][4] = image_t
self.cached_tokens[req.metadata_buffer_index][5] = audio_t
self.cached_tokens[req.metadata_buffer_index][6] = video_t
@@ -1180,6 +1180,7 @@ class Req(ReqDllmMixin):
self.mm_image_tokens: int = 0
self.mm_audio_tokens: int = 0
self.mm_video_tokens: int = 0
self.mm_token_counts_ready: bool = False
# Prefix info
# Tokens [0, prefix_len) are already in the row, or, before allocation,
@@ -1427,6 +1428,26 @@ class Req(ReqDllmMixin):
# Snapshot of the scheduler prefill-token counter taken at waiting_queue entry; used by HRRN aging.
self.arrival_processed_tokens: int = 0
def get_mm_token_counts(self) -> Tuple[int, int, int]:
"""Return per-modality prompt counts, scanning media at most once."""
if not self.mm_token_counts_ready and self.multimodal_inputs is not None:
(
self.mm_image_tokens,
self.mm_audio_tokens,
self.mm_video_tokens,
) = self.multimodal_inputs.compute_mm_token_counts()
self.mm_token_counts_ready = True
return self.mm_image_tokens, self.mm_audio_tokens, self.mm_video_tokens
def inherit_mm_token_counts(self, other: Req) -> None:
"""Reuse counts from the request whose media history is shared."""
(
self.mm_image_tokens,
self.mm_audio_tokens,
self.mm_video_tokens,
) = other.get_mm_token_counts()
self.mm_token_counts_ready = True
def advance_cache_request_handle(self) -> None:
self.cache_request_handle = dataclasses.replace(
self.cache_request_handle,
@@ -1519,6 +1540,13 @@ class Req(ReqDllmMixin):
self.spec_cap_lens_histogram[cap_len] += 1
def extend_image_inputs(self, image_inputs):
if self.mm_token_counts_ready:
image_tokens, audio_tokens, video_tokens = (
image_inputs.compute_mm_token_counts()
)
self.mm_image_tokens += image_tokens
self.mm_audio_tokens += audio_tokens
self.mm_video_tokens += video_tokens
if self.session is not None:
self._extend_session_image_inputs(image_inputs)
elif self.multimodal_inputs is None:
@@ -513,14 +513,7 @@ class _GenerationStreamAccumulator:
# already computed these and transferred them via the metadata buffer
# (req.mm_*), so prefer the pre-stored values; otherwise compute them
# from the request's multimodal items.
if req.mm_image_tokens or req.mm_audio_tokens or req.mm_video_tokens:
image_t = req.mm_image_tokens
audio_t = req.mm_audio_tokens
video_t = req.mm_video_tokens
elif req.multimodal_inputs:
image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts()
else:
image_t = audio_t = video_t = 0
image_t, audio_t, video_t = req.get_mm_token_counts()
self.image_tokens.append(image_t)
self.audio_tokens.append(audio_t)
self.video_tokens.append(video_t)
@@ -332,6 +332,9 @@ class Session:
)
if last_req is not None:
new_req.multimodal_inputs = last_req.multimodal_inputs
# An offset can truncate inherited media.
if not abort and not session_params.offset:
new_req.inherit_mm_token_counts(last_req)
new_req.tokenizer = tokenizer
if carry_fill is not None:
new_req.full_untruncated_fill_ids = carry_fill