From df966f234504523ab3ff4bd88521c99796405bc6 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 18 Oct 2026 00:07:55 -0700 Subject: [PATCH] [Feat] deepseek-v4.1: Run encoder SWA bounded replay on PD prefill workers (#44952) --- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 8 ++++- python/sglang/srt/disaggregation/prefill.py | 14 +++++++- python/sglang/srt/disaggregation/utils.py | 7 +++- .../srt/mem_cache/base_swa_memory_pool.py | 10 +++++- .../srt/mem_cache/deepseek_v4_memory_pool.py | 26 +++++++++----- .../srt/mem_cache/dsv41_request_window.py | 14 ++++++-- .../srt/model_executor/pool_configurator.py | 4 ++- .../mem_cache/test_dsv4_compressed_pools.py | 36 ++++++++++++++++++- .../model_executor/test_pool_configurator.py | 4 +-- 9 files changed, 104 insertions(+), 19 deletions(-) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 9dbc45d8cc..0b3ab79d40 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -297,7 +297,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: ("DP attention on ROCm", get_platform().is_hip and attn_dp_enabled_of(cfg)), ("external cache linker", cfg.enable_unified_cache_external_linker), ("unified memory", cfg.enable_unified_memory), - ("PD disaggregation", cfg.disaggregation_mode != "null"), + # Decode receives the last SWA window from prefill and never replays. + ("PD decode", cfg.disaggregation_mode == "decode"), + # On ROCm the draft keeps paged SWA, so it cannot share the window's pages. + ( + "PD prefill on ROCm", + cfg.disaggregation_mode == "prefill" and get_platform().is_hip, + ), ("mixed prefill/decode", cfg.enable_mixed_chunk), ("LoRA", cfg.enable_lora), ("radix sessions", cfg.enable_session_radix_cache), diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 0f6d8d2d22..a9926e2ac7 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1531,9 +1531,21 @@ class SchedulerDisaggregationPrefillMixin: ] def _swa_payload(): - window_size = self.sliding_window_size + kvcache = self.token_to_kv_pool_allocator.get_kvcache() + request_window = kvcache.request_window + # Under encoder replay the scheduler is not hybrid SWA and keeps no + # window size; the pool holds it. + window_size = ( + self.sliding_window_size + if request_window is None + else kvcache.sliding_window + ) window_start = max(req.disagg_decode_prefix_len, seq_len - window_size) window_start = (window_start // page_size) * page_size + if request_window is not None: + return request_window.page_indices( + int(req.kv.req_pool_idx), window_start, seq_len + ) window_kv_indices_full = self.req_to_token_pool.req_to_token[ req.kv.req_pool_idx, window_start:seq_len ] diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index b3090330d6..07f437d34a 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -1749,17 +1749,22 @@ def setup_state_kv_args( raise RuntimeError( "DSV4 target and draft pools must share the SWA index mapping" ) + # Under encoder replay the target window and the draft ring address SWA + # by request slot; equal rows per slot let them share one page list. + request_window = token_to_kv_pool.request_window target_geometry = ( token_to_kv_pool.page_size, token_to_kv_pool.sliding_window, + request_window.capacity if request_window is not None else None, ) draft_geometry = ( draft_token_to_kv_pool.page_size, draft_token_to_kv_pool.sliding_window, + draft_token_to_kv_pool.swa_req_ring_size, ) if target_geometry != draft_geometry: raise RuntimeError( - "DSV4 target and draft pools must share paged SWA geometry: " + "DSV4 target and draft pools must share SWA geometry: " f"target={target_geometry}, draft={draft_geometry}" ) draft_ptrs, draft_lens, draft_item_lens = ( diff --git a/python/sglang/srt/mem_cache/base_swa_memory_pool.py b/python/sglang/srt/mem_cache/base_swa_memory_pool.py index 3d381ab250..071002d9db 100644 --- a/python/sglang/srt/mem_cache/base_swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/base_swa_memory_pool.py @@ -1,10 +1,15 @@ +from __future__ import annotations + import abc -from typing import List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import torch from sglang.srt.mem_cache.memory_pool import KVCache +if TYPE_CHECKING: + from sglang.srt.mem_cache.dsv41_request_window import RequestWindow + class BaseSWAKVPool(KVCache): """ABC for SWA-like KV pools. @@ -18,6 +23,9 @@ class BaseSWAKVPool(KVCache): # Set when SWA KV is a per-request ring of this many tokens (addressed by # req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token. swa_req_ring_size: Optional[int] = None + # Set when the target keeps SWA KV in per-request window rows (DeepSeek-V4.1 + # encoder replay) instead of a paged pool; there is no full -> swa mapping. + request_window: Optional[RequestWindow] = None # Independent draft SWA buffers can consume full-token IDs directly instead # of the target SWA allocator's physical IDs. HiCache must preserve that # index ownership even though both buffers implement sliding attention. diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index ba2885a649..5cb1277257 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -130,9 +130,12 @@ def uses_draft_swa_ring() -> bool: return get_exec().features.enable_encoder_swa_bounded_replay and not _is_hip -def get_draft_swa_ring_size(sliding_window: int, max_draft_tokens: int) -> int: - # A draft block and a verify commit write past the committed window. - return sliding_window + max_draft_tokens +def get_draft_swa_ring_size( + sliding_window: int, max_draft_tokens: int, page_size: int +) -> int: + # A draft block and a verify commit write past the committed window; whole + # pages match the target request window's rows, so PD sends both as one list. + return ceil_div(sliding_window + max_draft_tokens, page_size) * page_size def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int: @@ -1169,7 +1172,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): # while attending, so it addresses its ring directly with no workspace. self.unified_kv_pool = None self.swa_req_ring_size = get_draft_swa_ring_size( - self.sliding_window, online_mtp_max_draft_tokens or 0 + self.sliding_window, online_mtp_max_draft_tokens or 0, swa_page_size ) self.swa_req_ring_valid_from = torch.zeros( self.num_req_slots, dtype=torch.int64, device=device @@ -1451,11 +1454,16 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): data_lens: List[int] = [] item_lens: List[int] = [] - if self.swa_kv_pool is not None: - physical_pages_per_logical_page = ( - self.swa_page_size // self.swa_kv_pool.page_size - ) - for buf in self.swa_kv_pool.kv_buffer: + # Under encoder replay the target's SWA lives in the request window, whose + # pages pair with decode's paged SWA pages (RequestWindow.page_indices). + swa_pool = ( + self.request_window.state + if self.request_window is not None + else self.swa_kv_pool + ) + if swa_pool is not None: + physical_pages_per_logical_page = self.swa_page_size // swa_pool.page_size + for buf in swa_pool.kv_buffer: assert buf.ndim == 2, f"expected 2D buffer, got {buf.ndim}D" data_ptrs.append(buf.data_ptr()) data_lens.append(buf.nbytes) diff --git a/python/sglang/srt/mem_cache/dsv41_request_window.py b/python/sglang/srt/mem_cache/dsv41_request_window.py index ea8cad7d4c..e0d254e2b1 100644 --- a/python/sglang/srt/mem_cache/dsv41_request_window.py +++ b/python/sglang/srt/mem_cache/dsv41_request_window.py @@ -213,6 +213,16 @@ class RequestWindow: size = ((rows + self.page_size - 1) // self.page_size) * self.page_size self.workspace = self.pool_factory(size, 1) + def row_loc(self, slots, positions): + return slots * self.capacity + positions % self.capacity + + def page_indices(self, slot: int, start: int, end: int): + assert start % self.page_size == 0, (start, self.page_size) + positions = torch.arange(start, end, self.page_size) + # capacity is whole pages, so a page's rows keep a paged pool's offsets + # and pair with decode's paged SWA pages one for one. + return (self.row_loc(slot, positions) // self.page_size).to(torch.int32).numpy() + def reset(self, slots): loc = slots.to(torch.int64)[:, None] * self.capacity + torch.arange( self.capacity, device=slots.device @@ -237,7 +247,7 @@ class RequestWindow: def initialize_dummy_history(self): layout = self.layout self.tags.fill_(-1) - loc = layout.history_req * self.capacity + layout.history_pos % self.capacity + loc = self.row_loc(layout.history_req, layout.history_pos) for buf in self.state.kv_buffer: buf.zero_() self.tags[:, loc] = layout.history_pos @@ -246,7 +256,7 @@ class RequestWindow: def _history_src(self, layout): return torch.where( layout.history_valid, - layout.history_req * self.capacity + layout.history_pos % self.capacity, + self.row_loc(layout.history_req, layout.history_pos), self.zero_row, ) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 568615d9fa..95b70b9f63 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -1200,7 +1200,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): + self.page_size ) draft_ring_size = get_draft_swa_ring_size( - self.sliding_window_size, self.online_c128_mtp_max_draft_tokens + self.sliding_window_size, + self.online_c128_mtp_max_draft_tokens, + self.page_size, ) self.request_window_bytes = ( (slots * capacity + self.page_size) * layers * (self.kv_bytes + 16) diff --git a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py index 281c6480f6..45aaac70a6 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py +++ b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py @@ -396,7 +396,7 @@ class TestV41KVPoolLayouts(CustomTestCase): class TestDSparkDraftRingWithEncoderReplay(CustomTestCase): - RING = 128 + 6 # sliding window + the largest draft token count + RING = 256 # sliding window + the largest draft token count, in whole pages def setUp(self): super().setUp() @@ -446,6 +446,40 @@ class TestDSparkDraftRingWithEncoderReplay(CustomTestCase): self.assertEqual(draft.swa_req_ring_size, self.RING) self.assertGreaterEqual(draft.swa_kv_pool.size, 3 * self.RING) + def test_pd_swa_state_pairs_with_a_paged_decode(self): + target = self.make_pool(draft=False) + draft = self.make_pool(draft=True) + self.assertEqual(target.request_window.capacity, draft.swa_req_ring_size) + override = get_context().override_server_args( + enable_encoder_swa_bounded_replay=False + ) + override.install() + try: + decode_target = self.make_pool(draft=False) + decode_draft = self.make_pool(draft=True) + finally: + override.restore() + for prefill, decode in ((target, decode_target), (draft, decode_draft)): + self.assertEqual( + prefill.get_state_buf_infos()[2], decode.get_state_buf_infos()[2] + ) + + def test_pd_window_pages_carry_the_last_window(self): + window = self.make_pool(draft=False).request_window + page, slot, sliding = window.page_size, 2, 128 + for seq_len in (300, 1000, 1030): + with self.subTest(seq_len=seq_len): + # A row holds the newest committed position with its residue. + held = torch.full((window.rows + page,), -1) + newest = torch.arange(seq_len - window.capacity, seq_len) + held[window.row_loc(slot, newest)] = newest + start = (seq_len - sliding) // page * page + pages = torch.from_numpy(window.page_indices(slot, start, seq_len)) + # Decode lays the received pages out by position from start. + received = held.view(-1, page)[pages.long()].flatten() + last = torch.arange(seq_len - sliding, seq_len) + self.assertEqual(received[last - start].tolist(), last.tolist()) + def _block_inputs(self, prefix_lens, slots, block_size=5): seq_lens_casual = build_block_seq_lens_causal( seq_lens=torch.tensor(prefix_lens), diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index d34f518c1d..37c2c2b1d0 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1539,10 +1539,10 @@ class TestSWAPoolFloor(CustomTestCase): paged = self._dspark_replay_planner(draft_ring=False) self.assertEqual(planner.bytes_per_swa_token, 0) self.assertEqual(planner.swa_cap_tokens, 0) - # Three draft stages of one (128 + 6)-row ring per request slot. + # Three draft stages of one ring per request slot: 128 + 6 rows in a page. self.assertEqual( planner.request_window_bytes - paged.request_window_bytes, - (3 * (128 + 6) + 256) * 3 * 584, + (3 * 256 + 256) * 3 * 584, ) sizes = planner.calculate_pool_sizes(256 * 1024 * 1024, 256) self.assertEqual(sizes.swa_max_total_num_tokens, 0)