[Core] Add request-level customized_info keys and mark Full prefill capture batches (#43772)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: jiayisuse <jiayisuse@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
jiayisuse
parent
149ef7d1b7
commit
5bcaf0f8ed
@@ -774,6 +774,7 @@ class TboForwardBatchPreparer:
|
||||
"spec_algorithm",
|
||||
"capture_hidden_mode",
|
||||
"defer_logits_to_eager", # forward-level flag, inherited by both child batches
|
||||
"full_prefill_cuda_graph_capture", # forward-level flag, inherited by both child batches
|
||||
"split_index", # for split prefill
|
||||
"orig_seq_lens", # only used by qwen-1m, thus not care
|
||||
"return_pooled_hidden_states",
|
||||
|
||||
@@ -173,6 +173,9 @@ class SchedulerOutputStreamer:
|
||||
default_stream_interval=get_serving().stream_interval,
|
||||
default_force_stream_interval=DEFAULT_FORCE_STREAM_INTERVAL,
|
||||
get_cached_tokens_details=self.get_cached_tokens_details,
|
||||
is_request_level_customized_info_key=(
|
||||
self.is_request_level_customized_info_key
|
||||
),
|
||||
rust_server_mode=self.rust_server is not None,
|
||||
current_weight_version=get_serving().weight_version,
|
||||
)
|
||||
@@ -230,6 +233,10 @@ class SchedulerOutputStreamer:
|
||||
"""Return whether this invocation needs subclass-provided output fields."""
|
||||
return True
|
||||
|
||||
def is_request_level_customized_info_key(self, key: str) -> bool:
|
||||
"""Whether a customized-info value is emitted once rather than token-sliced."""
|
||||
return False
|
||||
|
||||
def _maybe_log_time_stats(self, *, req: Req) -> None:
|
||||
if (
|
||||
req.finished()
|
||||
@@ -323,6 +330,7 @@ class _GenerationStreamAccumulator:
|
||||
default_stream_interval: int
|
||||
default_force_stream_interval: int
|
||||
get_cached_tokens_details: Callable[[Req], Optional[CachedTokensDetails]]
|
||||
is_request_level_customized_info_key: Callable[[str], bool]
|
||||
current_weight_version: Optional[str]
|
||||
rids: list = field(default_factory=list)
|
||||
output_reqs: list[Req] = field(default_factory=list)
|
||||
@@ -663,20 +671,33 @@ class _GenerationStreamAccumulator:
|
||||
current_output_len = len(self.output_ids[-1])
|
||||
if req.customized_info is not None:
|
||||
for key, req_values in req.customized_info.items():
|
||||
is_request_level = self.is_request_level_customized_info_key(key)
|
||||
if is_request_level:
|
||||
if req.stream:
|
||||
if send_token_offset != 0:
|
||||
continue
|
||||
elif not req.finished():
|
||||
continue
|
||||
if key not in self.customized_info:
|
||||
self.customized_info[key] = [
|
||||
[None] * len(prev_output_ids)
|
||||
[] if is_request_level else [None] * len(prev_output_ids)
|
||||
for prev_output_ids in self.output_ids[:-1]
|
||||
]
|
||||
self.customized_info[key].append(
|
||||
[None] * current_output_len
|
||||
if req_values is None
|
||||
else req_values[send_token_offset : len(output_ids_)]
|
||||
)
|
||||
if req_values is None:
|
||||
values = [] if is_request_level else [None] * current_output_len
|
||||
elif is_request_level:
|
||||
values = req_values
|
||||
else:
|
||||
values = req_values[send_token_offset : len(output_ids_)]
|
||||
self.customized_info[key].append(values)
|
||||
|
||||
for per_request_values in self.customized_info.values():
|
||||
for key, per_request_values in self.customized_info.items():
|
||||
if len(per_request_values) < len(self.output_ids):
|
||||
per_request_values.append([None] * current_output_len)
|
||||
per_request_values.append(
|
||||
[]
|
||||
if self.is_request_level_customized_info_key(key)
|
||||
else [None] * current_output_len
|
||||
)
|
||||
|
||||
def to_payload(
|
||||
self, *, dp_rank: int, is_idle_batch: bool
|
||||
|
||||
@@ -1940,6 +1940,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
)
|
||||
else:
|
||||
out = out_list[-1]
|
||||
if (
|
||||
is_stream
|
||||
and not incremental_stream
|
||||
and state.customized_info_accumulated
|
||||
):
|
||||
meta_info = out.get("meta_info")
|
||||
if meta_info is not None:
|
||||
for key, values in state.customized_info_accumulated.items():
|
||||
meta_info.setdefault(key, list(values))
|
||||
|
||||
# Resolve deferred text for non-incremental streaming.
|
||||
# _handle_batch_output sets "text": None on intermediate chunks
|
||||
|
||||
@@ -635,6 +635,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# CUDA-graph runners may capture only the transformer body and execute the
|
||||
# logits processor eagerly after replay.
|
||||
defer_logits_to_eager: bool = False
|
||||
# Full prefill graph capture uses a fixed request axis whose inactive slots
|
||||
# must not participate in data-dependent row selection.
|
||||
full_prefill_cuda_graph_capture: bool = False
|
||||
# For hidden states before normal
|
||||
return_hidden_states_before_norm: bool = False
|
||||
|
||||
|
||||
@@ -1604,6 +1604,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
lora_ids=([None] * bs if self._capture_lora else None),
|
||||
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
|
||||
max_seq_len_override=self.max_context_size,
|
||||
full_prefill_cuda_graph_capture=(
|
||||
self.prefill_backend_name == Backend.FULL
|
||||
),
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
ngram_manager = self.model_runner.ngram_embedding_manager
|
||||
|
||||
Reference in New Issue
Block a user