[Core] Add request-level customized_info keys and mark Full prefill capture batches (#43772)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: jiayisuse <jiayisuse@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:45:52 -07:00
committed by GitHub
co-authored by Hanming Lu jiayisuse
parent 149ef7d1b7
commit 5bcaf0f8ed
9 changed files with 210 additions and 12 deletions
@@ -774,6 +774,7 @@ class TboForwardBatchPreparer:
"spec_algorithm",
"capture_hidden_mode",
"defer_logits_to_eager", # forward-level flag, inherited by both child batches
"full_prefill_cuda_graph_capture", # forward-level flag, inherited by both child batches
"split_index", # for split prefill
"orig_seq_lens", # only used by qwen-1m, thus not care
"return_pooled_hidden_states",
@@ -173,6 +173,9 @@ class SchedulerOutputStreamer:
default_stream_interval=get_serving().stream_interval,
default_force_stream_interval=DEFAULT_FORCE_STREAM_INTERVAL,
get_cached_tokens_details=self.get_cached_tokens_details,
is_request_level_customized_info_key=(
self.is_request_level_customized_info_key
),
rust_server_mode=self.rust_server is not None,
current_weight_version=get_serving().weight_version,
)
@@ -230,6 +233,10 @@ class SchedulerOutputStreamer:
"""Return whether this invocation needs subclass-provided output fields."""
return True
def is_request_level_customized_info_key(self, key: str) -> bool:
"""Whether a customized-info value is emitted once rather than token-sliced."""
return False
def _maybe_log_time_stats(self, *, req: Req) -> None:
if (
req.finished()
@@ -323,6 +330,7 @@ class _GenerationStreamAccumulator:
default_stream_interval: int
default_force_stream_interval: int
get_cached_tokens_details: Callable[[Req], Optional[CachedTokensDetails]]
is_request_level_customized_info_key: Callable[[str], bool]
current_weight_version: Optional[str]
rids: list = field(default_factory=list)
output_reqs: list[Req] = field(default_factory=list)
@@ -663,20 +671,33 @@ class _GenerationStreamAccumulator:
current_output_len = len(self.output_ids[-1])
if req.customized_info is not None:
for key, req_values in req.customized_info.items():
is_request_level = self.is_request_level_customized_info_key(key)
if is_request_level:
if req.stream:
if send_token_offset != 0:
continue
elif not req.finished():
continue
if key not in self.customized_info:
self.customized_info[key] = [
[None] * len(prev_output_ids)
[] if is_request_level else [None] * len(prev_output_ids)
for prev_output_ids in self.output_ids[:-1]
]
self.customized_info[key].append(
[None] * current_output_len
if req_values is None
else req_values[send_token_offset : len(output_ids_)]
)
if req_values is None:
values = [] if is_request_level else [None] * current_output_len
elif is_request_level:
values = req_values
else:
values = req_values[send_token_offset : len(output_ids_)]
self.customized_info[key].append(values)
for per_request_values in self.customized_info.values():
for key, per_request_values in self.customized_info.items():
if len(per_request_values) < len(self.output_ids):
per_request_values.append([None] * current_output_len)
per_request_values.append(
[]
if self.is_request_level_customized_info_key(key)
else [None] * current_output_len
)
def to_payload(
self, *, dp_rank: int, is_idle_batch: bool
@@ -1940,6 +1940,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
)
else:
out = out_list[-1]
if (
is_stream
and not incremental_stream
and state.customized_info_accumulated
):
meta_info = out.get("meta_info")
if meta_info is not None:
for key, values in state.customized_info_accumulated.items():
meta_info.setdefault(key, list(values))
# Resolve deferred text for non-incremental streaming.
# _handle_batch_output sets "text": None on intermediate chunks
@@ -635,6 +635,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# CUDA-graph runners may capture only the transformer body and execute the
# logits processor eagerly after replay.
defer_logits_to_eager: bool = False
# Full prefill graph capture uses a fixed request axis whose inactive slots
# must not participate in data-dependent row selection.
full_prefill_cuda_graph_capture: bool = False
# For hidden states before normal
return_hidden_states_before_norm: bool = False
@@ -1604,6 +1604,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
lora_ids=([None] * bs if self._capture_lora else None),
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
max_seq_len_override=self.max_context_size,
full_prefill_cuda_graph_capture=(
self.prefill_backend_name == Backend.FULL
),
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
ngram_manager = self.model_runner.ngram_embedding_manager