[Perf] session: Cache multimodal prompt token counts across turns (#44112)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
joshuadeng
parent
9732188a58
commit
5f2861bf59
@@ -2629,6 +2629,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
decode_req.req.mm_image_tokens = cached_tokens[4].item()
|
||||
decode_req.req.mm_audio_tokens = cached_tokens[5].item()
|
||||
decode_req.req.mm_video_tokens = cached_tokens[6].item()
|
||||
decode_req.req.mm_token_counts_ready = True
|
||||
if not self.spec_algorithm.is_none():
|
||||
decode_req.req.output_topk_p = output_topk_p
|
||||
decode_req.req.output_topk_index = output_topk_index
|
||||
|
||||
@@ -549,10 +549,7 @@ class MetadataBuffers:
|
||||
|
||||
# Compute multimodal prompt token counts on the prefill node so decode
|
||||
# can report them in usage.
|
||||
if req.multimodal_inputs:
|
||||
image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts()
|
||||
else:
|
||||
image_t = audio_t = video_t = 0
|
||||
image_t, audio_t, video_t = req.get_mm_token_counts()
|
||||
self.cached_tokens[req.metadata_buffer_index][4] = image_t
|
||||
self.cached_tokens[req.metadata_buffer_index][5] = audio_t
|
||||
self.cached_tokens[req.metadata_buffer_index][6] = video_t
|
||||
|
||||
@@ -1180,6 +1180,7 @@ class Req(ReqDllmMixin):
|
||||
self.mm_image_tokens: int = 0
|
||||
self.mm_audio_tokens: int = 0
|
||||
self.mm_video_tokens: int = 0
|
||||
self.mm_token_counts_ready: bool = False
|
||||
|
||||
# Prefix info
|
||||
# Tokens [0, prefix_len) are already in the row, or, before allocation,
|
||||
@@ -1427,6 +1428,26 @@ class Req(ReqDllmMixin):
|
||||
# Snapshot of the scheduler prefill-token counter taken at waiting_queue entry; used by HRRN aging.
|
||||
self.arrival_processed_tokens: int = 0
|
||||
|
||||
def get_mm_token_counts(self) -> Tuple[int, int, int]:
|
||||
"""Return per-modality prompt counts, scanning media at most once."""
|
||||
if not self.mm_token_counts_ready and self.multimodal_inputs is not None:
|
||||
(
|
||||
self.mm_image_tokens,
|
||||
self.mm_audio_tokens,
|
||||
self.mm_video_tokens,
|
||||
) = self.multimodal_inputs.compute_mm_token_counts()
|
||||
self.mm_token_counts_ready = True
|
||||
return self.mm_image_tokens, self.mm_audio_tokens, self.mm_video_tokens
|
||||
|
||||
def inherit_mm_token_counts(self, other: Req) -> None:
|
||||
"""Reuse counts from the request whose media history is shared."""
|
||||
(
|
||||
self.mm_image_tokens,
|
||||
self.mm_audio_tokens,
|
||||
self.mm_video_tokens,
|
||||
) = other.get_mm_token_counts()
|
||||
self.mm_token_counts_ready = True
|
||||
|
||||
def advance_cache_request_handle(self) -> None:
|
||||
self.cache_request_handle = dataclasses.replace(
|
||||
self.cache_request_handle,
|
||||
@@ -1519,6 +1540,13 @@ class Req(ReqDllmMixin):
|
||||
self.spec_cap_lens_histogram[cap_len] += 1
|
||||
|
||||
def extend_image_inputs(self, image_inputs):
|
||||
if self.mm_token_counts_ready:
|
||||
image_tokens, audio_tokens, video_tokens = (
|
||||
image_inputs.compute_mm_token_counts()
|
||||
)
|
||||
self.mm_image_tokens += image_tokens
|
||||
self.mm_audio_tokens += audio_tokens
|
||||
self.mm_video_tokens += video_tokens
|
||||
if self.session is not None:
|
||||
self._extend_session_image_inputs(image_inputs)
|
||||
elif self.multimodal_inputs is None:
|
||||
|
||||
@@ -513,14 +513,7 @@ class _GenerationStreamAccumulator:
|
||||
# already computed these and transferred them via the metadata buffer
|
||||
# (req.mm_*), so prefer the pre-stored values; otherwise compute them
|
||||
# from the request's multimodal items.
|
||||
if req.mm_image_tokens or req.mm_audio_tokens or req.mm_video_tokens:
|
||||
image_t = req.mm_image_tokens
|
||||
audio_t = req.mm_audio_tokens
|
||||
video_t = req.mm_video_tokens
|
||||
elif req.multimodal_inputs:
|
||||
image_t, audio_t, video_t = req.multimodal_inputs.compute_mm_token_counts()
|
||||
else:
|
||||
image_t = audio_t = video_t = 0
|
||||
image_t, audio_t, video_t = req.get_mm_token_counts()
|
||||
self.image_tokens.append(image_t)
|
||||
self.audio_tokens.append(audio_t)
|
||||
self.video_tokens.append(video_t)
|
||||
|
||||
@@ -332,6 +332,9 @@ class Session:
|
||||
)
|
||||
if last_req is not None:
|
||||
new_req.multimodal_inputs = last_req.multimodal_inputs
|
||||
# An offset can truncate inherited media.
|
||||
if not abort and not session_params.offset:
|
||||
new_req.inherit_mm_token_counts(last_req)
|
||||
new_req.tokenizer = tokenizer
|
||||
if carry_fill is not None:
|
||||
new_req.full_untruncated_fill_ids = carry_fill
|
||||
|
||||
Reference in New Issue
Block a user