From a7fd2e1158c389c33077bae67fb797efa8dd09bd Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Sun, 18 Oct 2026 19:08:20 -0700 Subject: [PATCH] [Feat] dsv4.1: Return prompt logprobs under encoder SWA bounded replay (#45050) --- python/sglang/srt/managers/schedule_batch.py | 7 ---- .../sglang/srt/managers/tokenizer_manager.py | 8 ---- python/sglang/srt/managers/tp_worker.py | 11 ++++- .../srt/model_executor/encoder_swa_replay.py | 16 ++++---- .../model_executor/test_encoder_swa_replay.py | 41 ++++++++++++++++++- 5 files changed, 57 insertions(+), 26 deletions(-) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 7defe8566b..8ef50abfd7 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2875,13 +2875,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): "encoder SWA replay does not support input_embeds or " "positional_embed_overrides" ) - if req.return_logprob and req.logprob_start_len not in ( - -1, - len(req.origin_input_ids), - ): - raise ValueError( - "encoder SWA replay cannot return cached prompt logprobs" - ) self.encoder_swa_reset = [resets_swa_window(r) for r in reqs] # Allocate memory out_cache_loc, req_pool_indices_tensor, req_pool_indices_cpu = alloc_for_extend( diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index fc5ec64778..807eef50e4 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1342,14 +1342,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): "encoder SWA replay does not support video, audio, input_embeds, " "or positional_embed_overrides" ) - if ( - isinstance(obj, GenerateReqInput) - and obj.return_logprob - and obj.logprob_start_len not in (None, -1, len(input_ids)) - ): - raise ValueError( - "encoder SWA replay cannot return cached prompt logprobs" - ) # A decision layout indexes the whole prompt, so it is checked before any # truncation below could cut the prompt. self._validate_joint_schema_request(obj, input_ids) diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index bd962fec5b..ac83ca0815 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -663,7 +663,16 @@ class TpModelWorker(BaseTpWorker): drop_folded_rows, ) - drop_folded_rows(logits_output=logits_output, folded=folded) + tail = self.model_runner.attn_backend.tail_forward_metadata + drop_folded_rows( + logits_output=logits_output, + folded=folded, + tail_lens=( + None + if tail is None + else tail.late_layer_tail.extend_seq_lens_cpu + ), + ) batch_result = GenerationBatchResult( logits_output=logits_output, can_run_cuda_graph=can_run_cuda_graph, diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index fc6a6ac7e9..f5529abb1b 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -55,14 +55,16 @@ def apply_folded_extend(folded: FoldedExtend, forward_batch) -> None: forward_batch.encoder_swa_compress_rows = folded.keep_rows -def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None: +def drop_folded_rows( + *, logits_output, folded: FoldedExtend, tail_lens: Optional[list[int]] = None +) -> None: """Hand downstream consumers (the DSpark draft) only the original extend rows.""" hidden = logits_output.hidden_states token_indices = logits_output.hidden_states_token_indices if token_indices is not None: # Decoder-SWA tail rows index the folded extend; keep those on original rows. keep = _tail_rows_on_original( - folded=folded, num_tail_rows=token_indices.shape[0] + folded=folded, tail_lens=tail_lens, num_tail_rows=token_indices.shape[0] ).to(token_indices.device, non_blocking=True) logits_output.hidden_states_token_indices = folded.scheduled_row[ token_indices[keep] @@ -74,13 +76,11 @@ def drop_folded_rows(*, logits_output, folded: FoldedExtend) -> None: logits_output.hidden_states = hidden[folded.keep_rows] -def _tail_rows_on_original(*, folded: FoldedExtend, num_tail_rows: int): - # Each request's tail is its last SWA_WINDOW folded rows; replay rows lead it. - from sglang.srt.layers.attention.deepseek_v4_backend import SWA_WINDOW - +def _tail_rows_on_original( + *, folded: FoldedExtend, tail_lens: list[int], num_tail_rows: int +): keep, off = [], 0 - for n, r in zip(folded.batch.extend_lens, folded.replay_lens): - tail = min(SWA_WINDOW, n) + for n, r, tail in zip(folded.batch.extend_lens, folded.replay_lens, tail_lens): keep.append(torch.arange(off + max(0, tail - (n - r)), off + tail)) off += tail assert off == num_tail_rows, (off, num_tail_rows) diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 0f70008b1b..42366a67d1 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -18,6 +18,7 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( SWA_WINDOW, DeepseekV4AttnBackend, DSV4AttnMetadata, + late_layer_whole_extend, ) from sglang.srt.managers.mm_utils import pad_mm_tokens from sglang.srt.managers.schedule_batch import ( @@ -337,17 +338,18 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): replayed rows.""" fx = _Fixture() fb = fx.folded.batch - token_indices, _, _ = late_layer_tail_layout( + token_indices, tail_lens, _ = late_layer_tail_layout( extend_lens_cpu=list(fb.extend_lens), seq_lens_cpu=fx.folded_seq_lens(), tail_len=SWA_WINDOW, device=torch.device("cpu"), ) + self.assertEqual(tail_lens, [40, 128, 84]) out = SimpleNamespace( hidden_states=fb.input_ids[token_indices][:, None].float(), hidden_states_token_indices=token_indices, ) - drop_folded_rows(logits_output=out, folded=fx.folded) + drop_folded_rows(logits_output=out, folded=fx.folded, tail_lens=tail_lens) # Kept: all 40 miss rows, the hit's 30 new rows, the small hit's 20. expected = torch.cat( @@ -358,6 +360,41 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): out.hidden_states, fx.batch.input_ids[expected][:, None].float() ) + def test_prompt_logprob_reader_keeps_its_whole_tail(self): + """A prompt-logprob reader's tail is its whole folded extend, replay rows included.""" + fx = _Fixture() + fx.batch.extend_logprob_start_lens = [40, 10, 5] + folded = _fold_batch(batch=fx.batch, runner=fx.runner, rows=HITS) + fb = folded.batch + whole = late_layer_whole_extend( + SimpleNamespace( + extend_seq_lens_cpu=list(fb.extend_lens), + extend_logprob_start_lens_cpu=fb.extend_logprob_start_lens, + return_logprob=True, + capture_hidden_mode=CaptureHiddenMode.LAST, + ), + dspark=True, + ) + self.assertEqual(whole, [False, True, True]) + token_indices, tail_lens, _ = late_layer_tail_layout( + extend_lens_cpu=list(fb.extend_lens), + seq_lens_cpu=fx.folded_seq_lens(), + tail_len=SWA_WINDOW, + device=torch.device("cpu"), + whole_extend=whole, + ) + self.assertEqual(tail_lens, [40, 158, 84]) + out = SimpleNamespace( + hidden_states=fb.input_ids[token_indices][:, None].float(), + hidden_states_token_indices=token_indices, + ) + drop_folded_rows(logits_output=out, folded=folded, tail_lens=tail_lens) + + torch.testing.assert_close(out.hidden_states_token_indices, torch.arange(90)) + torch.testing.assert_close( + out.hidden_states, fx.batch.input_ids[:, None].float() + ) + def _extend_batch(fx): """The folded extend's forward batch, as the backend's metadata reads it."""