diff --git a/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py b/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py index 41d6996e0a..783c57fa6b 100644 --- a/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py @@ -443,15 +443,23 @@ def late_layer_tail_lens( extend_lens_cpu: list[int], tail_len: int, whole_extend: Optional[list[bool]] = None, + prompt_rest_lens: Optional[list[int]] = None, ) -> list[int]: """Each request's late-layer tail: its last min(tail_len, extend_len) tokens, - or its whole extend where whole_extend is set.""" - if whole_extend is None: - return [min(tail_len, n) for n in extend_lens_cpu] - return [ - n if whole else min(tail_len, n) - for n, whole in zip(extend_lens_cpu, whole_extend) - ] + or its whole extend where whole_extend is set. With prompt_rest_lens (prompt + tokens left after each extend), only its rows in the prompt's last tail_len + tokens, and at least its last row.""" + if whole_extend is not None and any(whole_extend): + return [ + n if whole else min(tail_len, n) + for n, whole in zip(extend_lens_cpu, whole_extend) + ] + if prompt_rest_lens is not None: + return [ + min(n, max(1, tail_len - rest)) + for n, rest in zip(extend_lens_cpu, prompt_rest_lens, strict=True) + ] + return [min(tail_len, n) for n in extend_lens_cpu] def late_layer_tail_layout( @@ -461,13 +469,15 @@ def late_layer_tail_layout( tail_len: int, device: torch.device, whole_extend: Optional[list[bool]] = None, + prompt_rest_lens: Optional[list[int]] = None, ) -> tuple[torch.Tensor, list[int], torch.Tensor]: """Tail rows of each prefill extend: its last min(tail_len, extend_len) tokens, or its whole extend where whole_extend is set. Returns (token indices into the extend, per-request tail lengths, per-row absolute window floor). The floor is seq_len - tail_len, so a tail shorter than tail_len reads rows the request wrote before this extend. A whole - extend keeps every row's plain window.""" + extend keeps every row's plain window. With prompt_rest_lens, tails and + floors follow the prompt's end instead (late_layer_tail_lens, _tail_floors).""" if whole_extend is not None and any(whole_extend): return _whole_extend_tail_layout( extend_lens_cpu=extend_lens_cpu, @@ -476,16 +486,19 @@ def late_layer_tail_layout( device=device, whole_extend=whole_extend, ) - tail_lens_cpu = late_layer_tail_lens(extend_lens_cpu, tail_len) + tail_lens_cpu = late_layer_tail_lens( + extend_lens_cpu, tail_len, prompt_rest_lens=prompt_rest_lens + ) + floors_cpu = _tail_floors(seq_lens_cpu, tail_len, prompt_rest_lens) if len(extend_lens_cpu) == 1: - n, t, s = extend_lens_cpu[0], tail_lens_cpu[0], seq_lens_cpu[0] - floor = torch.full((t,), max(0, s - tail_len), dtype=torch.int32, device=device) + n, t = extend_lens_cpu[0], tail_lens_cpu[0] + floor = torch.full((t,), floors_cpu[0], dtype=torch.int32, device=device) return torch.arange(n - t, n, device=device), tail_lens_cpu, floor # One H2D copy for the three length vectors; launch count does not grow with bs. lens = async_h2d( - [extend_lens_cpu, tail_lens_cpu, seq_lens_cpu], dtype=torch.int64, device=device + [extend_lens_cpu, tail_lens_cpu, floors_cpu], dtype=torch.int64, device=device ) - extend_lens, tail_lens, seq_lens = lens[0], lens[1], lens[2] + extend_lens, tail_lens, floors = lens[0], lens[1], lens[2] total = sum(tail_lens_cpu) req = torch.repeat_interleave( torch.arange(len(tail_lens_cpu), device=device), tail_lens, output_size=total @@ -495,10 +508,23 @@ def late_layer_tail_layout( - (torch.cumsum(tail_lens, 0) - tail_lens)[req] ) token_indices = (torch.cumsum(extend_lens, 0) - tail_lens)[req] + offs - floor = (seq_lens - tail_len).clamp_min(0)[req].to(torch.int32) + floor = floors[req].to(torch.int32) return token_indices, tail_lens_cpu, floor +def _tail_floors( + seq_lens_cpu: list[int], tail_len: int, prompt_rest_lens: Optional[list[int]] +) -> list[int]: + """Each request's window floor: tail_len below its extend's end, or below its + prompt's end, clamped to its last row, whose own KV a lone tail row reads.""" + if prompt_rest_lens is None: + return [max(0, s - tail_len) for s in seq_lens_cpu] + return [ + min(max(0, s + rest - tail_len), max(0, s - 1)) + for s, rest in zip(seq_lens_cpu, prompt_rest_lens, strict=True) + ] + + def _whole_extend_tail_layout( *, extend_lens_cpu: list[int], diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index d850cf399a..1af1a1ba1b 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -728,6 +728,7 @@ class TboForwardBatchPreparer: "extend_prefix_lens_cpu", "extend_seq_lens_cpu", "extend_logprob_start_lens_cpu", + "extend_prompt_rest_lens_cpu", "lora_ids", "rids", ]: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 415a16160b..07040b03eb 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1719,6 +1719,9 @@ class Envs: # Prefill-graph shapes of at least this many tokens run the tail from the tail # graphs in a break (measured crossover 1536-2048 tokens on GB300 TP4). SGLANG_DSV4_DECODER_TAIL_GRAPH_MIN_PREFILL_TOKENS = EnvInt(2048) + # Decoder SWA bounded replay: a prefill chunk runs the tail layers only over its + # rows in the prompt's last window, else over its last row. + SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL = EnvBool(False) # Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every # GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or # fp4 compressed caches), "v41_new" (the same bytes interleaved per token, for diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 5c62ddf18f..2882dd928e 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -1024,7 +1024,10 @@ def late_layer_num_tokens(forward_batch: ForwardBatch, *, dspark: bool) -> int: whole_extend = late_layer_whole_extend(forward_batch, dspark=dspark) return sum( late_layer_tail_lens( - forward_batch.extend_seq_lens_cpu, SWA_WINDOW, whole_extend + forward_batch.extend_seq_lens_cpu, + SWA_WINDOW, + whole_extend, + prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu, ) ) @@ -1834,6 +1837,7 @@ class DeepseekV4AttnBackend( whole_extend=late_layer_whole_extend( forward_batch, dspark=self.model_runner.spec_algorithm.is_dspark() ), + prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu, ) if self.encoder_row_floor is not None: # Folded hits: a tail row never reads below its request's replay start. diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index f663e80a2d..2f74303f0e 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -2252,6 +2252,7 @@ class DeepseekV4HipRadixBackend( tail_len=SWA_WINDOW, device=device, whole_extend=late_layer_whole_extend(forward_batch, dspark=self.is_dspark), + prompt_rest_lens=forward_batch.extend_prompt_rest_lens_cpu, ) contiguous_start = ( extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 41ca963bfe..d6f838f395 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -78,7 +78,7 @@ if TYPE_CHECKING: from sglang.srt.layers.engram_staging import StagedRows from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.logits_processor import LogitsProcessorOutput - from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch + from sglang.srt.managers.schedule_batch import MultimodalInputs, Req, ScheduleBatch from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo @@ -489,6 +489,17 @@ class NgramEmbeddingInfo: ) +def prompt_rest_lens_of( + reqs: List[Req], seq_lens: List[int], decoding_req_ids: Set[int] +) -> List[int]: + """Prompt tokens each request prefills after this extend, from the batch's own + ends (an encoder replay batch ends at its prefix); a decode row has none.""" + return [ + 0 if id(req) in decoding_req_ids else len(req.full_untruncated_fill_ids) - end + for req, end in zip(reqs, seq_lens, strict=True) + ] + + @dataclass class ForwardBatch(ForwardBatchDeepSeekMHAMixin): """Store all inputs of a forward pass.""" @@ -669,6 +680,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): extend_prefix_lens_cpu: Optional[List[int]] = None extend_seq_lens_cpu: Optional[List[int]] = None extend_logprob_start_lens_cpu: Optional[List[int]] = None + # Prompt tokens each request prefills after this extend (decoder SWA tail rows). + extend_prompt_rest_lens_cpu: Optional[List[int]] = None extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None # For DP attention (MLP sync sizes) @@ -1015,6 +1028,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): batch.seq_lens_sum = int(seq_lens_cpu.sum()) contains_last_prefill_chunk = False + prompt_rest_lens = None if batch.forward_mode.is_extend_without_speculative(): decoding_req_ids = {id(req) for req in (batch.decoding_reqs or ())} contains_last_prefill_chunk = any( @@ -1022,6 +1036,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): for req in batch.reqs if id(req) not in decoding_req_ids ) + if ( + envs.SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL.get() + and get_parallel().attn_cp_size == 1 + and seq_lens_cpu is not None + ): + prompt_rest_lens = prompt_rest_lens_of( + batch.reqs, seq_lens_cpu.tolist(), decoding_req_ids + ) from sglang.srt.model_executor.model_runner import ModelRunner @@ -1082,6 +1104,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): global_forward_mode=batch.global_forward_mode, is_prefill_only=batch.is_prefill_only, contains_last_prefill_chunk=contains_last_prefill_chunk, + extend_prompt_rest_lens_cpu=prompt_rest_lens, spec_algorithm=batch.spec_algorithm, capture_hidden_mode=capture_hidden_mode, return_hidden_states_before_norm=return_hidden_states_before_norm, @@ -2074,6 +2097,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self.extend_logprob_start_lens_cpu.extend( [0] * (bs - len(self.extend_logprob_start_lens_cpu)) ) + if self.extend_prompt_rest_lens_cpu is not None: + self.extend_prompt_rest_lens_cpu.extend( + [0] * (bs - len(self.extend_prompt_rest_lens_cpu)) + ) if self.rids_int is not None: self.rids_int = self._pad_tensor_to_size(self.rids_int, bs) diff --git a/test/registered/unit/layers/test_dsv41_prompt_window_tail.py b/test/registered/unit/layers/test_dsv41_prompt_window_tail.py new file mode 100644 index 0000000000..6d0b789389 --- /dev/null +++ b/test/registered/unit/layers/test_dsv41_prompt_window_tail.py @@ -0,0 +1,181 @@ +"""Decoder SWA bounded replay with SGLANG_OPT_DSV41_PROMPT_WINDOW_TAIL: a prefill +chunk runs the late layers over its rows in the prompt's last window only, so any +chunking of a prompt computes exactly the rows and floors of the unchunked prompt +there, plus one placeholder row per chunk outside it. CPU only.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( + late_layer_tail_layout, + late_layer_tail_lens, +) +from sglang.srt.layers.attention.deepseek_v4_backend import ( + SWA_WINDOW, + late_layer_num_tokens, +) +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardMode, + prompt_rest_lens_of, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +CPU = torch.device("cpu") + + +def _layout(extend_lens, seq_lens, rest=None, whole_extend=None): + return late_layer_tail_layout( + extend_lens_cpu=extend_lens, + seq_lens_cpu=seq_lens, + tail_len=SWA_WINDOW, + device=CPU, + whole_extend=whole_extend, + prompt_rest_lens=rest, + ) + + +def _chunked_rows(prompt_len, chunks): + """(absolute position, floor) of every tail row over a prompt's chunks, each + chunk one single-request step.""" + rows, start = [], 0 + for n in chunks: + end = start + n + token_indices, _, floor = _layout([n], [end], rest=[prompt_len - end]) + rows += list(zip((start + token_indices).tolist(), floor.tolist())) + start = end + assert start == prompt_len + return rows + + +class TestTailLens(CustomTestCase): + def test_final_chunks_keep_the_window_tail(self): + extend_lens = [300, 50, 128] + self.assertEqual( + late_layer_tail_lens(extend_lens, SWA_WINDOW, prompt_rest_lens=[0, 0, 0]), + late_layer_tail_lens(extend_lens, SWA_WINDOW), + ) + + def test_earlier_chunks_keep_their_rows_in_the_prompt_window(self): + # Far from the end: one row. Straddling the window: the rows inside it. + # Inside the window: the whole chunk. + self.assertEqual( + late_layer_tail_lens( + [32768, 300, 40, 300], + SWA_WINDOW, + prompt_rest_lens=[5000, 50, 20, SWA_WINDOW], + ), + [1, SWA_WINDOW - 50, 40, 1], + ) + + def test_whole_extend_wins(self): + self.assertEqual( + late_layer_tail_lens( + [300, 300], SWA_WINDOW, [True, False], prompt_rest_lens=[5000, 5000] + ), + [300, SWA_WINDOW], + ) + # No whole request: the prompt window applies, as in the layout. + self.assertEqual( + late_layer_tail_lens([300], SWA_WINDOW, [False], prompt_rest_lens=[5000]), + [1], + ) + + +class TestTailLayout(CustomTestCase): + def test_final_chunks_match_the_window_layout(self): + # Negative control: final chunks (nothing left) change no row or floor. + for extend_lens, seq_lens in (([300], [900]), ([300, 50, 40], [900, 50, 400])): + with self.subTest(extend_lens=extend_lens): + got = _layout(extend_lens, seq_lens, rest=[0] * len(extend_lens)) + want = _layout(extend_lens, seq_lens) + torch.testing.assert_close(got[0], want[0]) + self.assertEqual(got[1], want[1]) + torch.testing.assert_close(got[2], want[2]) + + def test_placeholder_row_reads_only_itself(self): + token_indices, tail_lens, floor = _layout([32768], [65536], rest=[70000]) + self.assertEqual(tail_lens, [1]) + self.assertEqual(token_indices.tolist(), [32767]) + self.assertEqual(floor.tolist(), [65535]) + + def test_batched_layout_matches_single_requests(self): + extend_lens = [32768, 300, 40, 200] + seq_lens = [40000, 900, 60, 200] + rest = [9000, 50, 20, 0] + token_indices, tail_lens, floor = _layout(extend_lens, seq_lens, rest) + want_indices, want_floor, start = [], [], 0 + for n, s, r in zip(extend_lens, seq_lens, rest): + idx, _, fl = _layout([n], [s], rest=[r]) + want_indices.append(start + idx) + want_floor.append(fl) + start += n + self.assertEqual(tail_lens, [1, SWA_WINDOW - 50, 40, SWA_WINDOW]) + torch.testing.assert_close(token_indices, torch.cat(want_indices)) + torch.testing.assert_close(floor, torch.cat(want_floor)) + + +class TestChunkingInvariance(CustomTestCase): + """The rows in the prompt's last window, and the floor each reads from, do + not depend on how the prompt was chunked.""" + + def test_any_chunking_computes_the_unchunked_window(self): + prompt_len = 1000 + whole = _chunked_rows(prompt_len, [prompt_len]) + floor = prompt_len - SWA_WINDOW + self.assertEqual(whole, [(p, floor) for p in range(floor, prompt_len)]) + for chunks in ( + [400, 400, 200], + [900, 60, 40], + [800, 100, 60, 40], + [872, 1, 127], + [700, 200, 100], + ): + with self.subTest(chunks=chunks): + rows = _chunked_rows(prompt_len, chunks) + window = [(p, f) for p, f in rows if p >= floor] + self.assertEqual(window, whole) + # Placeholders sit below the window and read only their own row. + for p, f in rows: + if p < floor: + self.assertEqual(f, p) + + def test_window_rows_never_read_below_the_window(self): + rows = _chunked_rows(1000, [900, 60, 40]) + self.assertTrue(all(f == 1000 - SWA_WINDOW for p, f in rows if p >= 872)) + + +class TestPromptRestLens(CustomTestCase): + def test_rest_from_the_batch_ends(self): + prompt = SimpleNamespace(full_untruncated_fill_ids=list(range(1000))) + decode = SimpleNamespace(full_untruncated_fill_ids=list(range(70))) + # A chunk ending at 400, a replay batch ending at its prefix 128, and a + # decode row carried in the extend. + self.assertEqual( + prompt_rest_lens_of([prompt, prompt, decode], [400, 128, 70], {id(decode)}), + [600, 872, 0], + ) + + def test_rows_counted_for_dp_match_the_layout(self): + extend_lens, rest = [32768, 300, 40], [9000, 50, 20] + fb = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + extend_seq_lens_cpu=extend_lens, + extend_logprob_start_lens_cpu=None, + return_logprob=False, + capture_hidden_mode=CaptureHiddenMode.FULL, + extend_prompt_rest_lens_cpu=rest, + ) + _, tail_lens, _ = _layout(extend_lens, [40000, 900, 60], rest) + self.assertEqual(late_layer_num_tokens(fb, dspark=True), sum(tail_lens)) + fb.extend_prompt_rest_lens_cpu = None + self.assertEqual(late_layer_num_tokens(fb, dspark=True), 2 * SWA_WINDOW + 40) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 4148234f17..dcba51ddd9 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -292,6 +292,7 @@ class TestFoldWithDecoderSwaTail(CustomTestCase): positions=fx.folded_positions(), capture_hidden_mode=CaptureHiddenMode.NULL, return_logprob=False, + extend_prompt_rest_lens_cpu=None, ) with ( patch.object(DSV4AttnMetadata, "init_compression_metadata"),