From 5bcaf0f8ede9fccbe63ffa1e29f51239e384460a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 13 Oct 2026 11:45:52 -0700 Subject: [PATCH] [Core] Add request-level customized_info keys and mark Full prefill capture batches (#43772) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: jiayisuse --- .../srt/batch_overlap/two_batch_overlap.py | 1 + .../scheduler_components/output_streamer.py | 37 ++++-- .../sglang/srt/managers/tokenizer_manager.py | 9 ++ .../srt/model_executor/forward_batch_info.py | 3 + .../runner/prefill_cuda_graph_runner.py | 3 + .../test_output_streamer_customized_info.py | 106 +++++++++++++++++- .../managers/test_output_streamer_logprobs.py | 1 + .../test_tokenizer_manager_rid_cleanup.py | 32 ++++++ .../test_prefill_cuda_graph_runner.py | 30 ++++- 9 files changed, 210 insertions(+), 12 deletions(-) diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index c4e457851e..457bc237da 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -774,6 +774,7 @@ class TboForwardBatchPreparer: "spec_algorithm", "capture_hidden_mode", "defer_logits_to_eager", # forward-level flag, inherited by both child batches + "full_prefill_cuda_graph_capture", # forward-level flag, inherited by both child batches "split_index", # for split prefill "orig_seq_lens", # only used by qwen-1m, thus not care "return_pooled_hidden_states", diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index 2b4e0d5b04..d2dd5a8253 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -173,6 +173,9 @@ class SchedulerOutputStreamer: default_stream_interval=get_serving().stream_interval, default_force_stream_interval=DEFAULT_FORCE_STREAM_INTERVAL, get_cached_tokens_details=self.get_cached_tokens_details, + is_request_level_customized_info_key=( + self.is_request_level_customized_info_key + ), rust_server_mode=self.rust_server is not None, current_weight_version=get_serving().weight_version, ) @@ -230,6 +233,10 @@ class SchedulerOutputStreamer: """Return whether this invocation needs subclass-provided output fields.""" return True + def is_request_level_customized_info_key(self, key: str) -> bool: + """Whether a customized-info value is emitted once rather than token-sliced.""" + return False + def _maybe_log_time_stats(self, *, req: Req) -> None: if ( req.finished() @@ -323,6 +330,7 @@ class _GenerationStreamAccumulator: default_stream_interval: int default_force_stream_interval: int get_cached_tokens_details: Callable[[Req], Optional[CachedTokensDetails]] + is_request_level_customized_info_key: Callable[[str], bool] current_weight_version: Optional[str] rids: list = field(default_factory=list) output_reqs: list[Req] = field(default_factory=list) @@ -663,20 +671,33 @@ class _GenerationStreamAccumulator: current_output_len = len(self.output_ids[-1]) if req.customized_info is not None: for key, req_values in req.customized_info.items(): + is_request_level = self.is_request_level_customized_info_key(key) + if is_request_level: + if req.stream: + if send_token_offset != 0: + continue + elif not req.finished(): + continue if key not in self.customized_info: self.customized_info[key] = [ - [None] * len(prev_output_ids) + [] if is_request_level else [None] * len(prev_output_ids) for prev_output_ids in self.output_ids[:-1] ] - self.customized_info[key].append( - [None] * current_output_len - if req_values is None - else req_values[send_token_offset : len(output_ids_)] - ) + if req_values is None: + values = [] if is_request_level else [None] * current_output_len + elif is_request_level: + values = req_values + else: + values = req_values[send_token_offset : len(output_ids_)] + self.customized_info[key].append(values) - for per_request_values in self.customized_info.values(): + for key, per_request_values in self.customized_info.items(): if len(per_request_values) < len(self.output_ids): - per_request_values.append([None] * current_output_len) + per_request_values.append( + [] + if self.is_request_level_customized_info_key(key) + else [None] * current_output_len + ) def to_payload( self, *, dp_rank: int, is_idle_batch: bool diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index b0fb98fbf0..8a50ee5454 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1940,6 +1940,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) else: out = out_list[-1] + if ( + is_stream + and not incremental_stream + and state.customized_info_accumulated + ): + meta_info = out.get("meta_info") + if meta_info is not None: + for key, values in state.customized_info_accumulated.items(): + meta_info.setdefault(key, list(values)) # Resolve deferred text for non-incremental streaming. # _handle_batch_output sets "text": None on intermediate chunks diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 3e6ac6df7a..39966191e2 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -635,6 +635,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # CUDA-graph runners may capture only the transformer body and execute the # logits processor eagerly after replay. defer_logits_to_eager: bool = False + # Full prefill graph capture uses a fixed request axis whose inactive slots + # must not participate in data-dependent row selection. + full_prefill_cuda_graph_capture: bool = False # For hidden states before normal return_hidden_states_before_norm: bool = False diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 242d0b23e6..2a0739c62a 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1604,6 +1604,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): lora_ids=([None] * bs if self._capture_lora else None), return_pooled_hidden_states=self.capture_return_pooled_hidden_states, max_seq_len_override=self.max_context_size, + full_prefill_cuda_graph_capture=( + self.prefill_backend_name == Backend.FULL + ), ) self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) ngram_manager = self.model_runner.ngram_embedding_manager diff --git a/test/registered/unit/managers/test_output_streamer_customized_info.py b/test/registered/unit/managers/test_output_streamer_customized_info.py index ec76d63694..483466e1f9 100644 --- a/test/registered/unit/managers/test_output_streamer_customized_info.py +++ b/test/registered/unit/managers/test_output_streamer_customized_info.py @@ -98,7 +98,11 @@ class _FakeReq: return False -def _accumulator(current_weight_version="default", return_sampling_mask=False): +def _accumulator( + current_weight_version="default", + return_sampling_mask=False, + request_level_keys=frozenset(), +): return _GenerationStreamAccumulator( return_logprob=False, return_hidden_states=False, @@ -110,6 +114,7 @@ def _accumulator(current_weight_version="default", return_sampling_mask=False): default_stream_interval=1, default_force_stream_interval=1, get_cached_tokens_details=lambda req: None, + is_request_level_customized_info_key=request_level_keys.__contains__, current_weight_version=current_weight_version, ) @@ -156,6 +161,105 @@ class TestOutputStreamerCustomizedInfo(unittest.TestCase): [[None, None], [None, None, None], [300]], ) + def test_request_level_value_is_preserved_for_finished_zero_token_request(self): + """A request that finishes with no output tokens still ships its value. + + Token-sliced keys yield ``req_values[0:0]`` here, so without the + request-level path the one value a prefill-only request produced + would arrive as an empty list. + """ + key = "prompt_score" + accumulator = _accumulator(request_level_keys=frozenset({key})) + + accumulator.accept( + req=_FakeReq( + "prefill-only", + [], + customized_info={key: [386.25]}, + finished=True, + ) + ) + + payload = accumulator.to_payload(dp_rank=0, is_idle_batch=False) + self.assertEqual( + unwrap_from_pickle(payload.customized_info), + {key: [[386.25]]}, + ) + + def test_request_level_customized_info_is_emitted_only_once(self): + """Streaming emits a request-level value with the first chunk only. + + A later chunk that re-sends it would be appended by the receiver and + the client would read the same value twice. + """ + key = "request_score" + req = _FakeReq("streaming", [10], customized_info={key: [386.25]}) + req.stream = True + + first = _accumulator(request_level_keys=frozenset({key})) + first.accept(req=req) + self.assertEqual( + unwrap_from_pickle( + first.to_payload(dp_rank=0, is_idle_batch=False).customized_info + ), + {key: [[386.25]]}, + ) + + req.output_ids.append(11) + second = _accumulator(request_level_keys=frozenset({key})) + second.accept(req=req) + self.assertIsNone( + second.to_payload(dp_rank=0, is_idle_batch=False).customized_info + ) + + def test_nonstreaming_request_level_customized_info_waits_for_finish(self): + """Non-streaming requests emit the value with the terminal output only. + + Intermediate outputs of a non-streaming request are coalesced on the + receiver, so emitting earlier would duplicate the value as well. + """ + key = "request_score" + req = _FakeReq("nonstreaming", [10], customized_info={key: [386.25]}) + + intermediate = _accumulator(request_level_keys=frozenset({key})) + intermediate.accept(req=req) + self.assertIsNone( + intermediate.to_payload(dp_rank=0, is_idle_batch=False).customized_info + ) + + req.output_ids.append(11) + req._finished = True + terminal = _accumulator(request_level_keys=frozenset({key})) + terminal.accept(req=req) + self.assertEqual( + unwrap_from_pickle( + terminal.to_payload(dp_rank=0, is_idle_batch=False).customized_info + ), + {key: [[386.25]]}, + ) + + def test_request_level_customized_info_uses_empty_list_for_missing_request(self): + """Requests without the key take an empty slot, not a ``None`` per token. + + The per-request lists stay aligned with ``output_ids`` while the + receiver never sees token-shaped padding for a request-level key. + """ + key = "request_score" + accumulator = _accumulator(request_level_keys=frozenset({key})) + + without_score = _FakeReq("without-score", [10, 11]) + without_score.stream = True + with_score = _FakeReq("with-score", [20], customized_info={key: [386.25]}) + with_score.stream = True + accumulator.accept(req=without_score) + accumulator.accept(req=with_score) + + payload = accumulator.to_payload(dp_rank=0, is_idle_batch=False) + self.assertEqual( + unwrap_from_pickle(payload.customized_info), + {key: [[], [386.25]]}, + ) + def test_additional_customized_info_uses_the_existing_payload(self): class Streamer(SchedulerOutputStreamer): has_additional_customized_info = True diff --git a/test/registered/unit/managers/test_output_streamer_logprobs.py b/test/registered/unit/managers/test_output_streamer_logprobs.py index 30b79283b7..d50736b0fb 100644 --- a/test/registered/unit/managers/test_output_streamer_logprobs.py +++ b/test/registered/unit/managers/test_output_streamer_logprobs.py @@ -74,6 +74,7 @@ def _make_accumulator() -> _GenerationStreamAccumulator: default_stream_interval=1, default_force_stream_interval=1, get_cached_tokens_details=lambda req: None, + is_request_level_customized_info_key=lambda key: False, current_weight_version=None, ) diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index 19750e5c81..d2258b6599 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -261,6 +261,38 @@ class TestEngineResponseWait(unittest.IsolatedAsyncioTestCase): with self.assertRaises(StopAsyncIteration): await anext(stream) + async def test_cumulative_stream_backlog_preserves_customized_info(self): + """Cumulative streaming returns only the last queued output. + + A customized_info key that arrived on an earlier queued output must + still reach the client through the accumulated state. + """ + self.tm.incremental_streaming_output = False + key = "request_score" + stream = self.tm._wait_one_response(self.state.obj) + self.state.customized_info_accumulated[key] = [386.25] + self.state.out_list.extend( + [ + { + "output_ids": [1], + "meta_info": {"finish_reason": None, key: [386.25]}, + }, + { + "output_ids": [1, 2], + "meta_info": {"finish_reason": {"type": "length"}}, + }, + ] + ) + self.state.finished = True + self.state.event.set() + + out = await anext(stream) + + self.assertEqual(out["output_ids"], [1, 2]) + self.assertEqual(out["meta_info"][key], [386.25]) + with self.assertRaises(StopAsyncIteration): + await anext(stream) + async def test_cancelling_pending_wait_removes_event_waiter(self): stream = self.tm._wait_one_response(self.state.obj) pending = asyncio.create_task(anext(stream)) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index f36d92caa6..e1aa38c653 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -181,7 +181,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): self.assertIs(runner.prepare_dummy_forward_batch(batch), batch) self.assertEqual(batch.attn_tp_sequence_sharded, expected) - def test_capture_prepare_applies_model_runner_batch_hook(self): + def _capture_runner(self, *, prefill_backend_name, capture_req_slots): class Slot: def __init__(self, buffer): self.buffer = buffer @@ -213,9 +213,9 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner.model_runner = model_runner runner.device = torch.device("cpu") - runner.prefill_backend_name = Backend.BREAKABLE + runner.prefill_backend_name = prefill_backend_name runner.max_context_size = None - runner._capture_req_slots = 1 + runner._capture_req_slots = capture_req_slots runner.max_bs = 4 runner._prefill_static_buffers = None runner.buffer_registry = registry @@ -231,6 +231,12 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner.tbo_plugin = SimpleNamespace( capture_one_batch_size=lambda batch, **_: captured.append(batch) ) + return runner, prepared, captured, attention_backend + + def test_capture_prepare_applies_model_runner_batch_hook(self): + runner, prepared, captured, attention_backend = self._capture_runner( + prefill_backend_name=Backend.BREAKABLE, capture_req_slots=1 + ) batch, backend = runner.capture_prepare(4) @@ -238,6 +244,24 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): self.assertEqual(captured, [batch]) self.assertIs(backend, attention_backend) + def test_capture_prepare_marks_full_prefill_capture(self): + """Only Full capture batches carry zero-length padding request slots. + + A model that selects rows per request reads the flag to keep those + slots out of the selection; a Breakable capture has no such slots. + """ + for backend_name, expected in ( + (Backend.FULL, True), + (Backend.BREAKABLE, False), + ): + with self.subTest(backend=backend_name): + runner, _, _, _ = self._capture_runner( + prefill_backend_name=backend_name, capture_req_slots=2 + ) + batch, _ = runner.capture_prepare(4) + self.assertEqual(batch.full_prefill_cuda_graph_capture, expected) + self.assertEqual(batch.batch_size, 2 if expected else 1) + def test_trim_logits_output_preserves_customized_info(self): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner.model_runner = SimpleNamespace(