diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_attn_metadata.py b/python/sglang/kernels/ops/speculative/dspark/dspark_attn_metadata.py index 52177dce2b..350d884127 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_attn_metadata.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_attn_metadata.py @@ -411,6 +411,83 @@ def build_dspark_swa_page_indices_triton( return swa_page_indices, swa_topk_lengths +def swa_ring_rows( + slots: torch.Tensor, positions: torch.Tensor, ring_size: int +) -> torch.Tensor: + """Row of ``positions`` in a per-request SWA ring: request slot ``s`` keeps + position ``p`` at row ``s * ring_size + p % ring_size``.""" + return slots.to(torch.int64) * ring_size + positions.to(torch.int64) % ring_size + + +class BuildDsparkSwaRingIndices: + """``BuildDsparkSwaPageIndices`` over a per-request SWA ring (`swa_ring_rows`), + with each request's window starting no earlier than its ``valid_from``.""" + + @classmethod + def execute(cls, *args, **kwargs) -> Tuple[torch.Tensor, torch.Tensor]: + return cls.torch(*args, **kwargs) + + @classmethod + def torch( + cls, + *, + seq_lens_casual: torch.Tensor, + req_pool_indices_repeated: torch.Tensor, + valid_from: torch.Tensor, + ring_size: int, + block_size: int, + swa_window: int, + page_index_aligned_size: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + num_q = seq_lens_casual.size(0) + assert num_q % block_size == 0, ( + f"DSpark draft block forward must be uniform-gamma: num_q={num_q} not " + f"divisible by block_size={block_size}." + ) + bs = num_q // block_size + device = seq_lens_casual.device + first_token = torch.arange(bs, device=device, dtype=torch.int64) * block_size + prefix_lens = seq_lens_casual[first_token].to(torch.int64) - 1 + slots = req_pool_indices_repeated[first_token].to(torch.int64).view(-1, 1) + start = torch.maximum( + prefix_lens - swa_window, valid_from[slots.view(-1)] + ).clamp_(min=0) + context_lens = (prefix_lens - start).clamp_(min=0) + + cols = torch.arange(swa_window, device=device, dtype=torch.int64).view(1, -1) + window_rows = swa_ring_rows(slots, start.view(-1, 1) + cols, ring_size) + window_rows = torch.where(cols < context_lens.view(-1, 1), window_rows, -1) + block_offsets = torch.arange(block_size, device=device, dtype=torch.int64) + block_rows = swa_ring_rows( + slots, prefix_lens.view(-1, 1) + block_offsets.view(1, -1), ring_size + ) + + target_width = ceil_align(swa_window + block_size, page_index_aligned_size) + out = torch.full((bs, target_width), -1, dtype=torch.int32, device=device) + out[:, :swa_window] = window_rows.to(torch.int32) + out.scatter_( + 1, + context_lens.view(-1, 1) + block_offsets.view(1, -1), + block_rows.to(torch.int32), + ) + + swa_page_indices = ( + out.view(bs, 1, target_width) + .expand(bs, block_size, target_width) + .reshape(bs * block_size, target_width) + .contiguous() + ) + swa_topk_lengths = ( + (context_lens + block_size) + .view(bs, 1) + .expand(bs, block_size) + .reshape(bs * block_size) + .contiguous() + .to(torch.int32) + ) + return swa_page_indices, swa_topk_lengths + + class BuildBlockSeqLensCausal: @classmethod def execute(cls, *args, **kwargs) -> torch.Tensor: diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py index b97077da2f..88f091a554 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py @@ -7,6 +7,7 @@ import triton.language as tl from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda +from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import swa_ring_rows from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout from sglang.srt.utils import ( @@ -620,7 +621,7 @@ class CommitInjectLayoutResult(msgspec.Struct): positions: torch.Tensor -def build_unified_commit_inject_layout( +def build_ring_commit_inject_layout( *, req_pool_indices: torch.Tensor, prefix_lens: torch.Tensor, @@ -629,13 +630,13 @@ def build_unified_commit_inject_layout( stride: int, ring_stride: int, ) -> CommitInjectLayoutResult: - """unified_kv counterpart of build_commit_inject_layout. + """Per-request SWA ring counterpart of build_commit_inject_layout. - Non-unified injection translates the verify tokens' full cache locs through - ``full_to_swa_mapping``; under unified_kv the SWA K lives in a ring addressed - directly by ``state_slot * ring_stride + pos % ring_stride``, so compute the - ring row here instead. Uncommitted tokens (col >= commit_len) get loc = -1 and - are skipped by the scatter. All ops are static-shape (CUDA-graph safe). + Paged injection translates the verify tokens' full cache locs through + ``full_to_swa_mapping``; a per-request ring is addressed directly by + `swa_ring_rows`, so compute the ring row here instead. Uncommitted tokens + (col >= commit_len) get loc = -1 and are skipped by the scatter. All ops are + static-shape (CUDA-graph safe). """ bs = req_pool_indices.shape[0] device = req_pool_indices.device @@ -644,7 +645,7 @@ def build_unified_commit_inject_layout( state_slot = ( req_pool_indices.to(torch.int64).view(-1, 1).expand(bs, stride).reshape(-1) ) - loc = state_slot * ring_stride + positions % ring_stride + loc = swa_ring_rows(state_slot, positions, ring_stride) col = torch.arange(stride, device=device).view(1, -1) committed = (col < commit_lens.to(torch.long).view(-1, 1)).reshape(-1) swa_loc = torch.where(committed, loc, torch.full_like(loc, -1)).to(torch.int32) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 4e3c266617..5eb17b471c 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -46,7 +46,9 @@ from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import ( BuildBlockSeqLensCausal, BuildDsparkSwaPageIndices, + BuildDsparkSwaRingIndices, ComputeDsparkWindowGather, + swa_ring_rows, ) from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import ( @@ -2311,7 +2313,10 @@ class DeepseekV4AttnBackend( self.topk, self.speculative_num_steps, )[self.speculative_step_id] - if self.token_to_kv_pool.request_window is None: + if ( + self.token_to_kv_pool.request_window is None + and self.token_to_kv_pool.swa_req_ring_size is None + ): metadata.core_attn_metadata.swa_out_cache_loc = ( self.token_to_kv_pool.translate_loc_from_full_to_swa( out_cache_loc @@ -2349,6 +2354,13 @@ class DeepseekV4AttnBackend( ) metadata.core_attn_metadata.swa_page_indices = swa_page_indices metadata.core_attn_metadata.swa_topk_lengths = swa_topk_lengths + if self.token_to_kv_pool.swa_req_ring_size is not None: + metadata.core_attn_metadata.swa_out_cache_loc = ( + self._dspark_swa_ring_out_cache_loc( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_pool_indices_repeated, + ) + ) if self.trtllm_attn: metadata.core_attn_metadata.init_trtllm_sparse_buffers() @@ -4276,6 +4288,7 @@ class DeepseekV4AttnBackend( raw_positions = prep.positions_casual request_layout = None + swa_out_cache_loc = None if self.token_to_kv_pool.request_window is not None: from sglang.srt.mem_cache.dsv41_request_window import window_layout @@ -4304,6 +4317,7 @@ class DeepseekV4AttnBackend( ) swa_page_indices = _pad_last_dim(request_layout.indices) swa_topk_lengths = request_layout.lengths + swa_out_cache_loc = request_layout.write_loc elif dspark_block_size is not None: assert ( self.is_dspark_draft @@ -4326,6 +4340,11 @@ class DeepseekV4AttnBackend( ) else: swa_page_indices, swa_topk_lengths = dspark_swa_buffers + if self.token_to_kv_pool.swa_req_ring_size is not None: + swa_out_cache_loc = self._dspark_swa_ring_out_cache_loc( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_pool_indices_repeated, + ) else: swa_page_indices = BuildCausalSwaPageIndices.execute( req_to_token=self.req_to_token, @@ -4359,9 +4378,7 @@ class DeepseekV4AttnBackend( present_ratios=self.present_ratios, low_ratios=self.low_ratios, request_window_layout=request_layout, - swa_out_cache_loc=( - request_layout.write_loc if request_layout is not None else None - ), + swa_out_cache_loc=swa_out_cache_loc, ) if need_compress: @@ -4395,6 +4412,17 @@ class DeepseekV4AttnBackend( out_loc: torch.Tensor, block_size: int, ) -> Tuple[torch.Tensor, torch.Tensor]: + pool = self.token_to_kv_pool + if pool.swa_req_ring_size is not None: + return BuildDsparkSwaRingIndices.execute( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_pool_indices_repeated, + valid_from=pool.swa_req_ring_valid_from, + ring_size=pool.swa_req_ring_size, + block_size=block_size, + swa_window=SWA_WINDOW, + page_index_aligned_size=PAGE_INDEX_ALIGNED_SIZE, + ) gather = ComputeDsparkWindowGather.execute( seq_lens_casual=seq_lens_casual, req_pool_indices_repeated=req_pool_indices_repeated, @@ -4416,6 +4444,17 @@ class DeepseekV4AttnBackend( ) return swa_page_indices, swa_topk_lengths + def _dspark_swa_ring_out_cache_loc( + self, *, seq_lens_casual: torch.Tensor, req_pool_indices_repeated: torch.Tensor + ) -> torch.Tensor: + """Ring rows the draft block writes: row ``i`` sits at position + ``seq_lens_casual[i] - 1`` of its request.""" + return swa_ring_rows( + req_pool_indices_repeated, + seq_lens_casual.to(torch.int64) - 1, + self.token_to_kv_pool.swa_req_ring_size, + ).to(torch.int32) + class DeepseekV4MultiStepBackend(DeepseekV4AttnBackend): def __init__( diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 054c2f4988..bcf09b82bf 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -112,6 +112,18 @@ def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int: return ring_size - window_size + 2 if ring_size > window_size else 0 +def uses_draft_swa_ring() -> bool: + """Whether the DSpark draft keeps its SWA KV in a per-request ring, so no SWA + state is tree-owned under encoder replay. The HIP backend still addresses the + draft SWA through the target's paged full-to-SWA mapping.""" + return get_exec().features.enable_encoder_swa_bounded_replay and not _is_hip + + +def get_draft_swa_ring_size(sliding_window: int, max_draft_tokens: int) -> int: + # A draft block and a verify commit write past the committed window. + return sliding_window + max_draft_tokens + + def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int: # A verify batch writes its draft tokens ahead of the committed position. spec_extra = (get_spec().speculative_num_draft_tokens - 1) if is_speculative else 0 @@ -1083,13 +1095,14 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): self.request_window = None self.full_to_swa_index_mapping = None + # Per req slot, the first position the draft SWA ring holds KV for. + self.swa_req_ring_valid_from: Optional[torch.Tensor] = None encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay - # DSpark's draft shares the target's full-to-SWA mapping, so the target - # keeps its paged SWA allocator even under encoder replay. - self.needs_paged_swa_allocator = ( - not encoder_replay - or is_draft_worker - or get_spec().speculative_algorithm is not None + # Under encoder replay the target keeps SWA in a per-request window. A + # draft without its own ring still reads the target's full-to-SWA mapping, + # so the target then keeps its paged SWA allocator. + self.needs_paged_swa_allocator = not encoder_replay or ( + not uses_draft_swa_ring() and get_spec().speculative_algorithm is not None ) if encoder_replay and not is_draft_worker: from sglang.srt.mem_cache.dsv41_request_window import RequestWindow @@ -1123,6 +1136,27 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): (self.num_req_slots + 1) * (1 + (online_mtp_max_draft_tokens or 0)), ), ) + elif uses_draft_swa_ring(): + # The DSpark draft writes SWA rows only from target hidden states, never + # while attending, so it addresses its ring directly with no workspace. + self.unified_kv_pool = None + self.swa_req_ring_size = get_draft_swa_ring_size( + self.sliding_window, online_mtp_max_draft_tokens or 0 + ) + self.swa_req_ring_valid_from = torch.zeros( + self.num_req_slots, dtype=torch.int64, device=device + ) + self.swa_kv_pool = self._make_kv_pool( + size=self.num_req_slots * self.swa_req_ring_size, + page_size=swa_kv_page_size, + dtype=dtype, + layer_num=stage_layer_num, + device=device, + enable_memory_saver=enable_memory_saver, + global_page_size=swa_page_size, + kv_layout=self.kv_layout, + cls=kv_pool_cls, + ) elif self._unified_kv: assert self.kv_layout is KVLayout.V4, ( "unified_kv keeps bf16 rows, not a paged FlashMLA layout" @@ -1169,10 +1203,16 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): cls=kv_pool_cls, ) + if self.request_window is not None: + swa_storage = "request_window" + elif self.swa_req_ring_size is not None: + swa_storage = "ring" + else: + swa_storage = "paged" logger.info( "DSV4 SWA storage: worker=%s, storage=%s, paged_allocator=%s", "draft" if is_draft_worker else "target", - "request_window" if self.request_window is not None else "paged", + swa_storage, self.needs_paged_swa_allocator, ) self.full_size = full_size @@ -1223,6 +1263,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): assert self.full_to_swa_index_mapping is not None return self.full_to_swa_index_mapping[kv_indices] + def reset_swa_req_ring(self, slots: torch.Tensor, valid_from: torch.Tensor) -> None: + """Restart the draft SWA ring of ``slots`` at ``valid_from``; the rows + before it belong to the slot's previous owner, not this request.""" + self.swa_req_ring_valid_from[slots] = valid_from + def get_contiguous_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: data_ptrs: List[int] = [] data_lens: List[int] = [] diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index bebd8277df..568615d9fa 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -34,9 +34,11 @@ from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( collect_sources_by_ratio, get_compress_state_write_pad, + get_draft_swa_ring_size, get_dsv4_indexer_bytes_per_token, get_swa_ring_size, resolve_compress_state_ring_size, + uses_draft_swa_ring, ) from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, @@ -1171,13 +1173,16 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): ) self.encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay - self.paged_draft_layers = 0 + draft_layers = 0 if self.encoder_replay and kvc.spec_algorithm.is_dspark(): aux = kvc.spec_aux_config - self.paged_draft_layers = int( + draft_layers = int( aux.dspark_num_stages or aux.dflash_draft_num_layers or 0 ) - assert self.paged_draft_layers > 0, "DSpark draft layer count is required" + assert draft_layers > 0, "DSpark draft layer count is required" + # A draft SWA ring is budgeted with the request windows, not per SWA token. + draft_ring_layers = draft_layers if uses_draft_swa_ring() else 0 + self.paged_draft_layers = draft_layers - draft_ring_layers self.request_window_bytes = 0 if self.encoder_replay: slots = self.requested_max_running_requests_per_worker + 1 @@ -1194,10 +1199,16 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): + slots * 128 + self.page_size ) + draft_ring_size = get_draft_swa_ring_size( + self.sliding_window_size, self.online_c128_mtp_max_draft_tokens + ) self.request_window_bytes = ( (slots * capacity + self.page_size) * layers * (self.kv_bytes + 16) + 4 * scratch * (self.kv_bytes + 16) + slots * 3 * 16 * self.attn_head_dim * 8 + + (slots * draft_ring_size + self.page_size) + * draft_ring_layers + * self.kv_bytes ) if not self.paged_draft_layers: self.swa_ratio = 0 diff --git a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py index 6606912e5c..24365707a4 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py @@ -2,12 +2,10 @@ from typing import Optional import torch -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( - is_unified_kv_triton, -) +from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import swa_ring_rows from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( BuildCommitInjectLayout, - build_unified_commit_inject_layout, + build_ring_commit_inject_layout, ) from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan @@ -32,6 +30,13 @@ class TargetHiddenKvInjector: self.verify_num_draft_tokens = verify_num_draft_tokens self._block_pos_offsets = block_pos_offsets + @property + def swa_ring_size(self) -> Optional[int]: + """The ring length when the draft's SWA KV is a per-request ring.""" + return getattr( + self.draft_model_runner.token_to_kv_pool, "swa_req_ring_size", None + ) + def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor: """The plan's write ids as the draft pool indexes them: the target's physical ids for a fused draft, the virtual ids for a private one.""" @@ -118,8 +123,8 @@ class TargetHiddenKvInjector: state_slot: Optional[torch.Tensor] = None, final_pos: Optional[torch.Tensor] = None, ) -> None: - if is_unified_kv_triton(): - swa_loc = self._unified_inject_loc( + if pool.swa_req_ring_size is not None: + swa_loc = self._ring_inject_loc( pool=pool, positions=positions, cache_loc_2d=cache_loc_2d, @@ -165,7 +170,7 @@ class TargetHiddenKvInjector: pool=pool, ) - def _unified_inject_loc( + def _ring_inject_loc( self, *, pool, @@ -175,9 +180,9 @@ class TargetHiddenKvInjector: state_slot: Optional[torch.Tensor], final_pos: Optional[torch.Tensor], ) -> torch.Tensor: - """Ring row for target-hidden injection under unified_kv. + """Ring row for target-hidden injection into a per-request SWA ring. - loc = state_slot * ring + pos % ring, with two skip (-1) rules: + loc = swa_ring_rows(state_slot, pos, ring), with two skip (-1) rules: * SWA window: only the last ``win`` tokens per req land in the ring; older tokens share a ring slot (pos % ring) and would race, so drop them (needed for long prefill chunks). @@ -185,13 +190,13 @@ class TargetHiddenKvInjector: """ if state_slot is None: raise RuntimeError( - "unified_kv target-hidden injection requires state_slot " + "SWA ring target-hidden injection requires state_slot " "(per-token draft req_pool_indices)." ) - ring = pool.unified_swa_ring_size - win = pool.unified_swa_window + ring = pool.swa_req_ring_size + win = pool.sliding_window pos = positions.to(torch.int64) - loc = state_slot.to(torch.int64) * ring + pos % ring + loc = swa_ring_rows(state_slot, pos, ring) if final_pos is not None: keep = pos > (final_pos.to(torch.int64) - win) loc = torch.where(keep, loc, torch.full_like(loc, -1)) @@ -220,14 +225,14 @@ class TargetHiddenKvInjector: if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"): if hidden_strided.numel() == 0: return - if is_unified_kv_triton(): - inject_layout = build_unified_commit_inject_layout( + if pool.swa_req_ring_size is not None: + inject_layout = build_ring_commit_inject_layout( req_pool_indices=batch.req_pool_indices, prefix_lens=prefix_lens, block_pos_offsets=self._block_pos_offsets[:stride], commit_lens=commit_lens, stride=stride, - ring_stride=pool.unified_swa_ring_size, + ring_stride=pool.swa_req_ring_size, ) else: inject_layout = BuildCommitInjectLayout.execute( diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index 273178cb92..2e3289cc24 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -6,9 +6,6 @@ from typing import Optional import msgspec import torch -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( - is_unified_kv_triton, -) from sglang.kernels.ops.speculative.dspark.dspark_accept import ( AcceptGreedy, AcceptSampling, @@ -22,7 +19,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( BuildRaggedVerifyWindow, RaggedVerifyWindow, ScatterCompactToStrided, - build_unified_commit_inject_layout, + build_ring_commit_inject_layout, scatter_compact_to_strided_into, ) from sglang.kernels.ops.speculative.dspark.simulated_bonus import ( @@ -369,8 +366,8 @@ class TargetVerifyExecutor: raise RuntimeError("DSpark verify requires target hidden states, got None.") hidden = hidden.view(bs, self.verify_num_draft_tokens, -1) state_slot = None - if is_unified_kv_triton(): - # unified_kv needs the per-token draft req slot to address the SWA ring + if self.kv_injector.swa_ring_size is not None: + # A SWA ring needs the per-token draft req slot to address it # (state_slot * ring + pos % ring). Verify tokens are the latest in each # req so they always fall in the window; the commit gate (via commit_lens # + cache_loc_2d) drops rejected tokens, so no final_pos skip is needed. @@ -659,7 +656,7 @@ class DsparkVerifyEpilogue: ) cache_loc = forward_batch.out_cache_loc state_slot = None - if is_unified_kv_triton(): + if self.commit_ctx.kv_injector.swa_ring_size is not None: state_slot = ( forward_batch.req_pool_indices.view(-1, 1) .expand(bs, self.stride) @@ -812,14 +809,14 @@ class DsparkVerifyEpilogue: torch.minimum(commit_lens, verify_lens.to(torch.int32)) * self.inject_gate_buf ) - if is_unified_kv_triton(): - inject_layout = build_unified_commit_inject_layout( + if pool.swa_req_ring_size is not None: + inject_layout = build_ring_commit_inject_layout( req_pool_indices=req_pool_indices, prefix_lens=seq_lens[:bs], block_pos_offsets=ctx.block_pos_offsets[: self.stride], commit_lens=gated_commit_lens, stride=self.stride, - ring_stride=pool.unified_swa_ring_size, + ring_stride=pool.swa_req_ring_size, ) else: inject_layout = BuildCommitInjectLayout.execute( diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 67f1c4c6a4..e9e9d0116a 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -3,9 +3,6 @@ from typing import TYPE_CHECKING, Callable, Optional, Protocol, runtime_checkabl import torch -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( - is_unified_kv_triton, -) from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import get_dp_tp_group @@ -698,12 +695,11 @@ class DSparkWorkerV2(BaseSpecWorker): ctx_lens, int(sum(batch.extend_lens)), ) - # unified_kv injects into the SWA ring keyed by (draft req slot, position); - # thread the per-token state_slot + the req's final position so the - # injector keeps only the last SWA window (older prefill tokens share a - # ring slot and would race). Cheap; only consumed under unified_kv. + # A SWA ring is keyed by (draft req slot, position); thread the per-token + # state_slot + the req's final position so the injector keeps only the + # last SWA window (older prefill tokens share a ring slot and would race). state_slot = final_pos = None - if is_unified_kv_triton(): + if self._kv_injector.swa_ring_size is not None: repeats = ctx_lens.to(torch.int64) num_tokens = sum(batch.extend_lens) state_slot = torch.repeat_interleave( @@ -716,6 +712,19 @@ class DSparkWorkerV2(BaseSpecWorker): repeats, output_size=num_tokens, ) + draft_pool = self._kv_injector.draft_model_runner.token_to_kv_pool + if getattr(draft_pool, "swa_req_ring_valid_from", None) is not None: + # A newly placed request's ring holds nothing below its cached prefix. + placed = [i for i, reset in enumerate(batch.encoder_swa_reset) if reset] + if placed: + draft_pool.reset_swa_req_ring( + batch.req_pool_indices[placed], + torch.tensor( + [batch.prefix_lens[i] for i in placed], + dtype=torch.int64, + device=device, + ), + ) # The draft KV goes to the slots the target prefill just wrote. cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan) batch_output.kv_loc_plan = None diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py index 2077891b24..9715559a11 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py @@ -121,7 +121,7 @@ class TestDSparkUnifiedKVKernelsAMD(CustomTestCase): self.assertIs(result, eager_metadata) backend.init_forward_metadata_target_verify_old.assert_called_once() - def test_build_unified_commit_inject_layout(self): + def test_build_ring_commit_inject_layout(self): stride, ring_stride = 7, 128 req_pool_indices = torch.tensor([3, 0, 5, 1], device=DEVICE, dtype=torch.int32) prefix_lens = torch.tensor( @@ -130,7 +130,7 @@ class TestDSparkUnifiedKVKernelsAMD(CustomTestCase): block_pos_offsets = torch.arange(stride, device=DEVICE, dtype=torch.int64) commit_lens = torch.tensor([0, 3, stride, 5], device=DEVICE, dtype=torch.int32) - got = dspark_verify_window.build_unified_commit_inject_layout( + got = dspark_verify_window.build_ring_commit_inject_layout( req_pool_indices=req_pool_indices, prefix_lens=prefix_lens, block_pos_offsets=block_pos_offsets, diff --git a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py index eee6b0cb4b..aee211099d 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py +++ b/test/registered/unit/mem_cache/test_dsv4_compressed_pools.py @@ -9,11 +9,16 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import ( KVLayout, is_valid_kv_layout_pair, ) +from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import ( + BuildDsparkSwaRingIndices, + build_block_seq_lens_causal, + build_dspark_swa_page_indices, + compute_dspark_window_gather, +) from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( SparsePrefillChunkCache, ) -from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( DeepSeekV4SingleKVPool, DeepSeekV4TokenToKVPool, @@ -23,6 +28,9 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import get_context +from sglang.srt.speculative.dspark_components.dspark_kv_inject import ( + TargetHiddenKvInjector, +) from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -382,7 +390,9 @@ class TestV41KVPoolLayouts(CustomTestCase): self.assertEqual(min(rows), c4_kv_rows) -class TestPagedDSparkWithEncoderReplay(CustomTestCase): +class TestDSparkDraftRingWithEncoderReplay(CustomTestCase): + RING = 128 + 6 # sliding window + the largest draft token count + def setUp(self): super().setUp() override = get_context().override_server_args( @@ -423,19 +433,99 @@ class TestPagedDSparkWithEncoderReplay(CustomTestCase): is_draft_worker=draft, ) - def test_target_window_and_draft_paged_storage_share_allocator_mapping(self): + def test_no_swa_state_needs_the_paged_allocator(self): target = self.make_pool(draft=False) draft = self.make_pool(draft=True) - allocator = SWATokenToKVPoolAllocator( - 2048, 1024, 256, torch.float8_e4m3fn, "cpu", target, False + self.assertFalse(target.needs_paged_swa_allocator) + self.assertFalse(draft.needs_paged_swa_allocator) + self.assertEqual(draft.swa_req_ring_size, self.RING) + self.assertGreaterEqual(draft.swa_kv_pool.size, 3 * self.RING) + + def _block_inputs(self, prefix_lens, slots, block_size=5): + seq_lens_casual = build_block_seq_lens_causal( + seq_lens=torch.tensor(prefix_lens), + block_size=block_size, + device="cpu", ) - draft.register_mapping(allocator.full_to_swa_index_mapping) - allocator.full_to_swa_index_mapping[256:512] = torch.arange(768, 1024) + req_pool_indices_repeated = torch.tensor(slots).repeat_interleave(block_size) + return seq_lens_casual, req_pool_indices_repeated + + def test_ring_indices_match_paged_indices_over_ring_rows(self): + prefix_lens, slots, block_size = [5, 200, 300], [2, 0, 1], 5 + seq_lens_casual, req_rep = self._block_inputs(prefix_lens, slots) + # A paged mapping that sends every (slot, position) to its ring row. + positions = torch.arange(512) + req_to_token = torch.arange(3).view(-1, 1) * 1000 + positions + mapping = torch.full((3000,), -1, dtype=torch.int64) + for slot in range(3): + mapping[req_to_token[slot]] = slot * self.RING + positions % self.RING + gather = compute_dspark_window_gather( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_rep, + block_size=block_size, + swa_window=128, + ) + out_loc = torch.cat( + [req_to_token[s, p : p + block_size] for s, p in zip(slots, prefix_lens)] + ) + paged = build_dspark_swa_page_indices( + req_to_token=req_to_token, + full_to_swa_mapping=mapping, + req_pool_indices_per_request=gather.req_pool_indices_per_request, + offsets=gather.offsets, + invalid=gather.invalid, + out_loc=out_loc, + context_lens=gather.context_lens, + block_size=block_size, + swa_window=128, + page_index_aligned_size=64, + ) + ring = BuildDsparkSwaRingIndices.torch( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_rep, + valid_from=torch.zeros(3, dtype=torch.int64), + ring_size=self.RING, + block_size=block_size, + swa_window=128, + page_index_aligned_size=64, + ) + for got, want in zip(ring, paged): + torch.testing.assert_close(got, want) + + def test_ring_window_starts_at_valid_from(self): + draft = self.make_pool(draft=True) + # Slot 1 was placed at a 290-token prefix hit; 10 positions are its own. + draft.reset_swa_req_ring(torch.tensor([1]), torch.tensor([290])) + seq_lens_casual, req_rep = self._block_inputs([300], [1]) + indices, lengths = BuildDsparkSwaRingIndices.torch( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_rep, + valid_from=draft.swa_req_ring_valid_from, + ring_size=self.RING, + block_size=5, + swa_window=128, + page_index_aligned_size=64, + ) + self.assertEqual(lengths.tolist(), [15] * 5) + rows = [self.RING + p % self.RING for p in range(290, 305)] + self.assertEqual(indices[0, :15].tolist(), rows) + self.assertTrue((indices[:, 15:] == -1).all()) + + def test_injection_writes_only_the_last_window(self): + pool = SimpleNamespace(swa_req_ring_size=self.RING, sliding_window=128) + positions = torch.arange(300) + loc = TargetHiddenKvInjector._ring_inject_loc( + None, + pool=pool, + positions=positions, + cache_loc_2d=None, + commit_lens=None, + state_slot=torch.full((300,), 2), + final_pos=torch.full((300,), 299), + ) + self.assertTrue((loc[:172] == -1).all()) self.assertEqual( - draft.translate_loc_from_full_to_swa( - torch.tensor([256, 300, 511]) - ).tolist(), - [768, 812, 1023], + loc[172:].tolist(), (2 * self.RING + positions[172:] % self.RING).tolist() ) diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index ff05a1e256..d34f518c1d 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1492,7 +1492,7 @@ class TestSWAPoolFloor(CustomTestCase): 0.1 * cfg.bytes_per_swa_token + 585 / 4 + 864 / 128 + 132 / 4, ) - def test_dsv4_paged_dspark_budget_reserves_window_and_draft_layers(self): + def _dspark_replay_planner(self, *, draft_ring): from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator _publish_config( @@ -1528,7 +1528,27 @@ class TestSWAPoolFloor(CustomTestCase): dflash_draft_num_layers=40, dspark_num_stages=3 ), ) - planner = DSV4PoolConfigurator(kvc) + with patch( + "sglang.srt.model_executor.pool_configurator.uses_draft_swa_ring", + return_value=draft_ring, + ): + return DSV4PoolConfigurator(kvc) + + def test_dsv4_dspark_draft_ring_budget_is_fixed(self): + planner = self._dspark_replay_planner(draft_ring=True) + paged = self._dspark_replay_planner(draft_ring=False) + self.assertEqual(planner.bytes_per_swa_token, 0) + self.assertEqual(planner.swa_cap_tokens, 0) + # Three draft stages of one (128 + 6)-row ring per request slot. + self.assertEqual( + planner.request_window_bytes - paged.request_window_bytes, + (3 * (128 + 6) + 256) * 3 * 584, + ) + sizes = planner.calculate_pool_sizes(256 * 1024 * 1024, 256) + self.assertEqual(sizes.swa_max_total_num_tokens, 0) + + def test_dsv4_paged_dspark_budget_reserves_window_and_draft_layers(self): + planner = self._dspark_replay_planner(draft_ring=False) self.assertEqual(planner.bytes_per_swa_token, 3 * 584) budget = 256 * 1024 * 1024 sizes = planner.calculate_pool_sizes(budget, 256)