diff --git a/python/sglang/kernels/ops/attention/extend_attention.py b/python/sglang/kernels/ops/attention/extend_attention.py index dd37f56111..4ca5962001 100644 --- a/python/sglang/kernels/ops/attention/extend_attention.py +++ b/python/sglang/kernels/ops/attention/extend_attention.py @@ -559,7 +559,9 @@ def _fwd_kernel( mask=(mask_m[:, None] & mask_n[None, :]), other=0, ) - final_mask &= custom_mask + # The mask buffer is int8; narrow it to i1, since a score_mod hands + # final_mask to tl.load, which rejects an i8 mask. + final_mask &= custom_mask != 0 if SLIDING_WINDOW_SIZE > 0: # Add mask where q_id <= kv_id + sliding_window_size # q_id = prefix_len + cur_m, kv_id = cur_n @@ -749,7 +751,8 @@ def _fwd_kernel( mask=(mask_m[:, None] & mask_n[None, :]), other=0, ) - custom_mask &= mask_m[:, None] & mask_n[None, :] + # int8 -> i1 before the &=; see the prefix loop above. + custom_mask = (custom_mask != 0) & mask_m[:, None] & mask_n[None, :] final_mask &= custom_mask elif IS_CAUSAL: mask_causual = (cur_block_m * BLOCK_M + offs_m[:, None]) >= ( @@ -1349,7 +1352,8 @@ def _fwd_kernel_unified( mask=(mask_m[:, None] & mask_n[None, :]), other=0, ) - final_mask &= custom_mask + # int8 -> i1 before the &=; see `_fwd_kernel`'s prefix loop. + final_mask &= custom_mask != 0 # Apply causal mask for extend part if IS_CAUSAL and not USE_CUSTOM_MASK: diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index 8fa7b5c2f2..1dfa822ad8 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -17,6 +17,7 @@ from sglang.kernels.spec import ( _TRITON_KERNELS = [ ("cache_locs", "assign_extend_cache_locs_func"), ("cache_locs", "generate_draft_decode_kv_indices"), + ("cache_locs", "generate_draft_decode_window_kv_indices"), ("eagle", "fill_bonus_tokens"), ("eagle", "fill_accept_out_cache_loc"), ("gather_spec_extras", "gather_spec_extras"), diff --git a/python/sglang/kernels/ops/speculative/cache_locs.py b/python/sglang/kernels/ops/speculative/cache_locs.py index 03c8c241a4..9ed4ecbbee 100644 --- a/python/sglang/kernels/ops/speculative/cache_locs.py +++ b/python/sglang/kernels/ops/speculative/cache_locs.py @@ -265,6 +265,124 @@ def generate_draft_decode_kv_indices( tl.store(kv_indptr + zid, base + zid * iters) +@triton.jit +def generate_draft_decode_window_kv_indices( + req_pool_indices, + req_to_token, + paged_kernel_lens, + window_kv_indices, + window_kv_indptr, + positions, + # Runtime, as in generate_draft_decode_kv_indices. + row_stride, + kv_indices_stride: tl.constexpr, + kv_indptr_stride: tl.constexpr, + bs_upper: tl.constexpr, + iter_upper: tl.constexpr, + num_tokens_upper: tl.constexpr, + page_size: tl.constexpr, + window: tl.constexpr, + ENTRY_PAGE_SIZE: tl.constexpr = 1, + # The source's sliding-window page table (`KVIndexTable.v2p`). + v2p=None, + TRANSLATE: tl.constexpr = False, +): + """Per-step sliding-window read rail of the multi-step draft. + + Step ``iters`` of branch ``(bid, topk_id)`` attends ``seq_len + iters`` + tokens: the committed prefix plus the draft's own chain in the slots + `generate_draft_decode_kv_indices` emits. A window layer reads the last + ``min(seq_len + iters, window)`` of them, so this emits that tail only, + prefix ids first and chain ids after, from the sliding-window read source. + """ + BLOCK_SIZE: tl.constexpr = 128 + iters = tl.program_id(axis=0) + bid = tl.program_id(axis=1) + topk_id = tl.program_id(axis=2) + + num_steps = tl.num_programs(axis=0) + num_seqs = tl.num_programs(axis=1) + topk = tl.num_programs(axis=2) + + window_kv_indices += kv_indices_stride * iters + window_kv_indptr += kv_indptr_stride * iters + iters += 1 + + seq_len = tl.load(paged_kernel_lens + bid) + total = seq_len + iters + window_len = tl.minimum(total, window) + # Chain position of the first token in the window; the prefix part covers + # [start, seq_len), the chain part [max(start - seq_len, 0), iters). + start = total - window_len + + # This branch's output offset: the window lengths of the branches before + # it, each clipped like this one. + zid = bid * topk + topk_id + bs_offset = tl.arange(0, num_tokens_upper) + in_front = bs_offset < zid + prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0) + prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0) + base = tl.sum(prev_window) + out_ptr = window_kv_indices + base + token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride + + prefix_start = tl.minimum(start, seq_len) + prefix_len = seq_len - prefix_start + kv_offset = tl.arange(0, BLOCK_SIZE) + num_loop = tl.cdiv(prefix_len, BLOCK_SIZE) + for _ in range(num_loop): + mask = kv_offset < prefix_len + data = _load_token_ids( + token_pool_ptr, + prefix_start + kv_offset, + mask, + v2p, + page_size, + ENTRY_PAGE_SIZE, + TRANSLATE, + ) + tl.store(out_ptr + kv_offset, data, mask=mask) + kv_offset += BLOCK_SIZE + + chain_start = tl.maximum(start - seq_len, 0) + extend_offset = tl.arange(0, iter_upper) + chain_step = chain_start + extend_offset + chain_mask = chain_step < iters + # The same chain slots `generate_draft_decode_kv_indices` emits. + if page_size == 1 or topk == 1: + chain_pos = seq_len + topk_id * num_steps + chain_step + else: + last_page_len = seq_len % page_size + num_new_pages_per_topk = ( + last_page_len + num_steps + page_size - 1 + ) // page_size + prefix_base = seq_len // page_size * page_size + chain_base = ( + prefix_base + topk_id * num_new_pages_per_topk * page_size + last_page_len + ) + chain_pos = chain_base + chain_step + extend_data = _load_token_ids( + token_pool_ptr, + chain_pos, + chain_mask, + v2p, + page_size, + ENTRY_PAGE_SIZE, + TRANSLATE, + ) + tl.store(out_ptr + prefix_len + extend_offset, extend_data, mask=chain_mask) + + # Update window_kv_indptr: program 0 writes the total at [num_seqs * topk], + # every other program its own start; [0] stays the buffer's zero. + if zid == 0: + zid = num_seqs * topk + in_front = bs_offset < zid + prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0) + prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0) + base = tl.sum(prev_window) + tl.store(window_kv_indptr + zid, base) + + @triton.jit def align_evict_mask_to_page_size( seq_lens, diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 6c5ec13d92..24c6e1a001 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -607,7 +607,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: assert _mc.is_hybrid_swa or mambaish_config(_mc) is not None, ( "--enable-unified-memory + EAGLE/EAGLE3 requires a unified " "target (hybrid-SWA or a mamba hybrid): the draft's KV lives " - "fused inside the full-attention page envelope." + "fused inside the target's sub-pool entries." ) # The target verifies on its own pages: the MLA family on an MLA host. eagle_allowed = ( diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index d6c5be56ac..6006a5a902 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -998,6 +998,12 @@ class ModelConfig: and self.hf_config.architectures[0] == "InklingForConditionalGeneration" ): self.hf_config.architectures[0] = "InklingForConditionalGenerationMTP" + if self.hf_text_config.mtp_local_layer_ids: + # A local MTP depth keeps the trunk's window unless it declares + # its own: the draft backend builds its window rail only if set. + self.sliding_window_size = ( + self.hf_text_config.mtp_local_extent or self.sliding_window_size + ) if ( is_draft_model and self.hf_config.architectures[0] == "GigaChat35ForCausalLM" diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 547b6f87bd..3d4ae2676e 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -1,5 +1,6 @@ from __future__ import annotations +import contextlib from dataclasses import dataclass from typing import TYPE_CHECKING, List, Optional @@ -41,10 +42,12 @@ from sglang.srt.model_executor.cuda_graph_config import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule, get_spec +from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc from sglang.srt.speculative.spec_utils import ( draft_kv_indices_buffer_width, draft_kv_indices_used_len, generate_draft_decode_kv_indices, + generate_draft_decode_window_kv_indices, resolve_draft_decode_window, ) from sglang.srt.utils import ( @@ -719,6 +722,22 @@ class TritonAttnBackend(AttentionBackend): kv_indptr = self._fill_kv_indptr_and_indices( plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices ) + if self.sliding_window_size is not None and self.sliding_window_size > 0: + # A window layer reads the window-clipped prefix, as the eager + # draft-extend path builds it. + _, _, _, self.cuda_graph_window_kv_offsets[:bs] = ( + update_sliding_window_buffer( + self.window_kv_indptr, + self.kv_index_translator, + plan, + req_pool_indices, + self.sliding_window_size, + kv_lens, + bs, + token_to_kv_pool=self.token_to_kv_pool, + window_kv_indices=self.cuda_graph_window_kv_indices, + ) + ) return qo_indptr, kv_indptr, num_tokens_per_req def init_forward_metadata_out_graph( @@ -744,6 +763,19 @@ class TritonAttnBackend(AttentionBackend): # Multi-step spec decode: kv buffers come from spec_info, not the # cuda-graph pool, so replay is not involved. if forward_mode.is_decode_or_idle() and spec_info is not None: + # This step's swa write ids, in the buffer the multi-step + # backend refills from the step's columns before replay. + swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc( + forward_batch, in_capture=True + ) + assert not ( + self.use_sliding_window_kv_pool and swa_out_cache_loc is None + ), ( + "multi-step spec decode on a sliding-window KV pool could " + "not derive swa_out_cache_loc; the window layers' writes " + "have no target. The per-step out_cache_loc must fit " + "cuda_graph_swa_out_cache_loc." + ) self.forward_metadata = ForwardMetadata( attn_logits=self.cuda_graph_attn_logits, attn_lse=self.cuda_graph_attn_lse, @@ -754,11 +786,18 @@ class TritonAttnBackend(AttentionBackend): qo_indptr=None, custom_mask=None, mask_indptr=None, - window_kv_indptr=self.window_kv_indptr, - window_kv_indices=None, + # The per-step window rail lives in the multi-step + # backend's persistent buffers, refilled before replay. + window_kv_indptr=( + spec_info.window_kv_indptr + if spec_info.window_kv_indices is not None + else self.window_kv_indptr + ), + window_kv_indices=spec_info.window_kv_indices, window_num_kv_splits=None, window_kv_offsets=None, swa_attn_logits=self.cuda_graph_swa_attn_logits, + swa_out_cache_loc=swa_out_cache_loc, lean_Mp=self.cuda_graph_lean_Mp, lean_Lp=self.cuda_graph_lean_Lp, lean_Op=self.cuda_graph_lean_Op, @@ -800,8 +839,9 @@ class TritonAttnBackend(AttentionBackend): self, forward_batch: ForwardBatch, in_capture: bool = False ) -> Optional[torch.Tensor]: """Refill the SWA write-target buffer with the batch's write ids in the - sliding-window sub-pool, returning the [:n] view (None for non-SWA / - multi-step draft) so the captured store reads fresh slots on replay. + sliding-window sub-pool, returning the [:n] view (None for non-SWA) so + the captured store reads fresh slots on replay. A multi-step draft step + hands in a batch bound to its own columns of the window. """ if not self.use_sliding_window_kv_pool: return None @@ -892,6 +932,18 @@ class TritonAttnBackend(AttentionBackend): else: kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices bs = kv_indptr.shape[0] - 1 + if spec_info.window_kv_indices is not None: + # The multi-step draft backend built the per-step window + # rail (the draft's own chain, window-clipped). + window_kv_indptr = spec_info.window_kv_indptr + window_kv_indices = spec_info.window_kv_indices + window_num_kv_splits = torch.empty( + (bs,), dtype=torch.int32, device=self.device + ) + self.get_num_kv_splits( + window_num_kv_splits, + window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs], + ) attn_logits = torch.empty( (bs, self.num_head, self.max_kv_splits, self.v_head_dim), @@ -1317,10 +1369,13 @@ class TritonAttnBackend(AttentionBackend): qo_indptr=self.qo_indptr[: bs + 1], custom_mask=None, mask_indptr=None, - window_kv_indptr=self.window_kv_indptr, - window_kv_indices=None, + window_kv_indptr=( + self.window_kv_indptr[: bs + 1] if swa else self.window_kv_indptr + ), + window_kv_indices=self.cuda_graph_window_kv_indices if swa else None, + # Extend reads through qo_indptr, never through kv splits. window_num_kv_splits=None, - window_kv_offsets=None, + window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None, swa_out_cache_loc=swa_out_cache_loc, ) else: @@ -1750,6 +1805,13 @@ class TritonAttnBackend(AttentionBackend): kv_indptr = self.forward_metadata.window_kv_indptr kv_indices = self.forward_metadata.window_kv_indices window_kv_offsets = self.forward_metadata.window_kv_offsets + # A null read rail reaches Triton as `None + offsets` and fails as + # an unreadable IR-build error; say what is actually missing. + assert kv_indices is not None, ( + f"layer {layer.layer_id} is a sliding-window layer but this " + "extend forward built no window_kv_indices; every mode that " + "reaches forward_extend must fill the window rail." + ) else: sliding_window_size = -1 kv_indptr = self.forward_metadata.kv_indptr @@ -2234,6 +2296,18 @@ class TritonAttnBackend(AttentionBackend): logits_soft_cap = logit_capping_mod(layer.logit_capping_method, layer.logit_cap) if save_kv_cache: + swa_loc = self.forward_metadata.swa_out_cache_loc + if ( + self.is_draft_runner + and forward_batch.spec_info is not None + and swa_loc is not None + ): + # KVWriteLoc keeps a longer swa_loc's prefix, but a draft step's + # slice of the window is not a prefix of it. + assert swa_loc.shape[0] == forward_batch.out_cache_loc.shape[0], ( + f"draft step swa_out_cache_loc has {swa_loc.shape[0]} " + f"entries for {forward_batch.out_cache_loc.shape[0]} tokens" + ) if self.use_mla: if layer.k_scale is not None: # MLATokenToKVPool doesn't accept scale parameters; k is unused @@ -2265,6 +2339,14 @@ class TritonAttnBackend(AttentionBackend): if layer.sliding_window_size is not None and layer.sliding_window_size > -1: kv_indptr = self.forward_metadata.window_kv_indptr kv_indices = self.forward_metadata.window_kv_indices + # A null read rail reaches Triton as `None + offsets` and fails as + # an unreadable IR-build error; say what is actually missing. + assert kv_indices is not None, ( + f"layer {layer.layer_id} is a sliding-window layer but this " + "forward built no window_kv_indices; a multi-step draft gets the " + "per-step window rail whenever its model has a window " + "(TritonMultiStepDraftBackend.window_rail)." + ) else: kv_indptr = self.forward_metadata.kv_indptr kv_indices = self.forward_metadata.kv_indices @@ -2451,15 +2533,35 @@ class TritonMultiStepDraftBackend: model_runner ) self.kv_index_translator = model_runner.kv_index_translator + self.token_to_kv_pool = model_runner.token_to_kv_pool + # Each draft step's window layers read a window-clipped rail of their + # own: the draft's chain is not part of the committed prefix yet. + self.sliding_window_size = model_runner.sliding_window_size + self.window_rail = ( + self.sliding_window_size is not None and self.sliding_window_size > 0 + ) + self.window_kv_indptr = ( + torch.zeros( + (self.speculative_num_steps, max_bs + 1), + dtype=torch.int32, + device=model_runner.device, + ) + if self.window_rail + else None + ) + self.cuda_graph_window_kv_indices = None def common_template( self, forward_batch: ForwardBatch, kv_indices_buffer: Optional[torch.Tensor], call_fn: int, + window_kv_indices_buffer: Optional[torch.Tensor] = None, ): if kv_indices_buffer is None: kv_indices_buffer = self.cuda_graph_kv_indices + if window_kv_indices_buffer is None: + window_kv_indices_buffer = self.cuda_graph_window_kv_indices num_seqs = forward_batch.batch_size bs = self.topk * num_seqs @@ -2496,6 +2598,8 @@ class TritonMultiStepDraftBackend: v2p=src.v2p, TRANSLATE=src.v2p is not None, ) + if self.window_rail: + self._fill_window_rail(forward_batch, window_kv_indices_buffer, bs) if call_fn is None: return @@ -2505,8 +2609,95 @@ class TritonMultiStepDraftBackend: forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ : draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1) ] + if self.window_rail: + forward_batch.spec_info.window_kv_indptr = self.window_kv_indptr[ + i, : bs + 1 + ] + forward_batch.spec_info.window_kv_indices = window_kv_indices_buffer[i][ + : bs * self.sliding_window_size + ] call_fn(i, forward_batch) + @contextlib.contextmanager + def _step_writes(self, forward_batch, step: int): + """Bind draft step ``step``'s slice of the batch's write ids, as + `EagleWorkerV2.draft_forward` slices them, and its columns of the + iteration's write window, while a per-step backend derives its write + targets (the swa ids included) from the batch.""" + loc, cols = forward_batch.out_cache_loc, forward_batch.kv_loc_cols + if loc is None or not self.attn_backends[step].use_sliding_window_kv_pool: + yield + return + num_steps = self.speculative_num_steps + if cols is None: + # The draft's own window, each row laid out (topk, num_steps). + step_cols = slice(step, None, num_steps) + else: + # A chain draft writes the leading columns of the verify window. + assert self.topk == 1 and isinstance(cols, slice) and cols.step is None + step_cols = slice(cols.start + step, cols.start + step + 1) + forward_batch.out_cache_loc = per_step_draft_out_cache_loc( + loc, forward_batch.batch_size, self.topk, num_steps + )[step] + forward_batch.kv_loc_cols = step_cols + try: + yield + finally: + forward_batch.out_cache_loc, forward_batch.kv_loc_cols = loc, cols + + def _fill_window_rail( + self, + forward_batch: ForwardBatch, + window_kv_indices_buffer: torch.Tensor, + bs: int, + ) -> None: + """Emit every step's window-clipped read ids into the per-step rows + of ``window_kv_indices_buffer`` and their indptr into + ``self.window_kv_indptr``. The reader selects FULL or SWA storage; + the layer's window determines which positions to gather.""" + num_seqs = forward_batch.batch_size + width = bs * self.sliding_window_size + assert window_kv_indices_buffer.shape[1] >= width, ( + f"window rail row {window_kv_indices_buffer.shape[1]} < " + f"{bs} branch(es) x window {self.sliding_window_size}" + ) + src = self.kv_index_translator.read_source( + forward_batch.kv_loc_plan, + req_pool_indices=forward_batch.req_pool_indices, + bs=num_seqs, + kind=self.kv_index_translator.window_kv_kind, + ) + generate_draft_decode_window_kv_indices[ + (self.speculative_num_steps, num_seqs, self.topk) + ]( + src.row_ids, + src.ids, + forward_batch.seq_lens, + window_kv_indices_buffer, + self.window_kv_indptr, + forward_batch.positions, + src.row_stride, + window_kv_indices_buffer.shape[1], + self.window_kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(self.speculative_num_steps), + next_power_of_2(bs), + self.page_size, + self.sliding_window_size, + ENTRY_PAGE_SIZE=src.entry_page_size, + v2p=src.v2p, + TRANSLATE=src.v2p is not None, + ) + if ( + not src.is_translated + and self.kv_index_translator.window_kv_kind == IdSpaceKind.SLIDING_WINDOW + ): + # Static pool: the legacy full->swa translate, as + # update_sliding_window_buffer applies it to the target's window. + for i in range(self.speculative_num_steps): + row = window_kv_indices_buffer[i][:width] + row.copy_(self.token_to_kv_pool.translate_loc_from_full_to_swa(row)) + def init_forward_metadata(self, forward_batch: ForwardBatch): kv_indices_width = draft_kv_indices_buffer_width( forward_batch.batch_size, self.topk, self.max_context_len @@ -2516,6 +2707,17 @@ class TritonMultiStepDraftBackend: dtype=torch.int64, device=self.device, ) + window_kv_indices = None + if self.window_rail: + # zeros, not empty: a static pool translates the whole row. + window_kv_indices = torch.zeros( + ( + self.speculative_num_steps, + forward_batch.batch_size * self.topk * self.sliding_window_size, + ), + dtype=torch.int64, + device=self.device, + ) def call_fn(i, forward_batch): forward_batch.spec_info.kv_indptr = ( @@ -2524,9 +2726,22 @@ class TritonMultiStepDraftBackend: forward_batch.spec_info.kv_indices = ( forward_batch.spec_info.kv_indices.clone() ) - self.attn_backends[i].init_forward_metadata(forward_batch) + if self.window_rail: + forward_batch.spec_info.window_kv_indptr = ( + forward_batch.spec_info.window_kv_indptr.clone() + ) + forward_batch.spec_info.window_kv_indices = ( + forward_batch.spec_info.window_kv_indices.clone() + ) + with self._step_writes(forward_batch, i): + self.attn_backends[i].init_forward_metadata(forward_batch) - self.common_template(forward_batch, kv_indices, call_fn) + self.common_template( + forward_batch, + kv_indices, + call_fn, + window_kv_indices_buffer=window_kv_indices, + ) def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): kv_indices_width = draft_kv_indices_buffer_width( @@ -2543,6 +2758,15 @@ class TritonMultiStepDraftBackend: dtype=torch.int32, device=self.device, ) + if self.window_rail: + self.cuda_graph_window_kv_indices = torch.zeros( + ( + self.speculative_num_steps, + max_bs * self.topk * self.sliding_window_size, + ), + dtype=torch.int64, + device=self.device, + ) for i in range(self.speculative_num_steps - 1): self.attn_backends[i].init_cuda_graph_state( @@ -2567,9 +2791,10 @@ class TritonMultiStepDraftBackend: ) def call_fn(i, _forward_batch): - self.attn_backends[i].init_forward_metadata_out_graph( - inner_fb, in_capture=True - ) + with self._step_writes(inner_fb, i): + self.attn_backends[i].init_forward_metadata_out_graph( + inner_fb, in_capture=True + ) self.common_template(forward_batch, None, call_fn) else: @@ -2587,6 +2812,14 @@ class TritonMultiStepDraftBackend: self.attn_backends[-1].cuda_graph_num_kv_splits[:num_token], forward_batch.seq_lens[:bs], ) + if self.attn_backends[0].use_sliding_window_kv_pool: + # Each captured step writes its window layers through its own + # backend's swa buffer: refill it from that step's columns. + for i in range(self.speculative_num_steps - 1): + with self._step_writes(forward_batch, i): + self.attn_backends[i]._fill_cuda_graph_swa_out_cache_loc( + forward_batch + ) def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None: for attn_backend in self.attn_backends: @@ -2640,12 +2873,7 @@ def update_sliding_window_buffer( indptr=window_kv_indptr, out=window_kv_indices, kv_start_idx=window_kv_start_idx, - kind=( - IdSpaceKind.SLIDING_WINDOW - if translator.reads_are_translated - and isinstance(token_to_kv_pool, BaseSWAKVPool) - else IdSpaceKind.FULL - ), + kind=translator.window_kv_kind, token_mapping=token_mapping, ) if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool): diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 92c8ead68b..3e83f8e787 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3976,7 +3976,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): def maybe_evict_swa(self): if self.tree_cache.supports_swa(): - sliding_window_size = self.tree_cache.sliding_window_size + retain_window = self.tree_cache.swa_retain_window # Auxiliary windows check their own cursors and prefix locks. has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa() @@ -3985,9 +3985,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get()) - swa_evict_due = self._swa_eviction_trigger( - sliding_window_size, eviction_interval - ) + swa_evict_due = self._swa_eviction_trigger(retain_window, eviction_interval) self.token_to_kv_pool_allocator.free_group_begin() for idx, req in enumerate(self.reqs): if self.forward_mode.is_decode(): @@ -4009,12 +4007,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ), ) - # Past the window the request no longer reads its prefill's SWA; - # release that part of the tree lock so SWA LRU can reclaim it. - if ( - release_leaf_lock - and req.decode_batch_idx >= sliding_window_size - ): + # Past the retain window the request no longer reads its prefill's + # SWA; release that part of the tree lock so SWA LRU can reclaim it. + if release_leaf_lock and req.decode_batch_idx >= retain_window: self.tree_cache.release_swa_prefix_lock(req) elif ( self.forward_mode.is_extend() diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index c5824e0e68..6f783d667b 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -590,6 +590,11 @@ class Scheduler( if self.draft_worker is not None else None ), + draft_swa_window=( + self.draft_worker.draft_swa_window + if self.draft_worker is not None + else 0 + ), ) self.is_hybrid_swa = result.is_hybrid_swa self.is_hybrid_ssm = result.is_hybrid_ssm diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 0cbb696c84..e290a0ec35 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -391,6 +391,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait): storage_prefetch_retries: Optional[StoragePrefetchRetries] = None # Set by caches that publish KV placement events; None means they don't. kv_events: Optional[KVCacheEventRecorder] = None + # Tokens back from the committed length a fused draft reads in the swa + # sub-pool (CacheInitParams.draft_swa_window); 0 when it reads none. + draft_swa_window: int = 0 def init_metrics_collector(self): from sglang.srt.layers.dp_attention import is_dp_attention_enabled @@ -663,6 +666,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def supports_auxiliary_swa(self) -> bool: return False + @property + def swa_retain_window(self) -> Optional[int]: + # What the frees and unlocks of a running request's SWA keep behind it; + # every other SWA rule uses sliding_window_size. + if self.sliding_window_size is None: + return None + return max(self.sliding_window_size, self.draft_swa_window) + def evict_sliding_windows( self, req: Req, pre_len: int, *, eviction_interval: int = 1 ) -> None: @@ -672,7 +683,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait): free_swa_out_of_window_slots( req, pre_len, - sliding_window_size=self.sliding_window_size, + sliding_window_size=self.swa_retain_window, page_size=self.page_size, req_to_token_pool=self.req_to_token_pool, token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index af56ae0543..ae4f1790a3 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -58,6 +58,9 @@ class CacheInitParams: # paged SWA pool only a draft uses), so it must not gate prefix matching. swa_gates_prefix_match: bool = True decoder_swa_bounded_replay: bool = False + # Tokens back from the committed length that a fused draft reads in the + # swa sub-pool (BaseSpecWorker.draft_swa_window); 0 when it reads none. + draft_swa_window: int = 0 # Time-to-live for cache entries in seconds. If None, TTL is disabled. cache_ttl_seconds: Optional[float] = None @@ -70,3 +73,12 @@ class CacheInitParams: mtp_draft_device_pools: tuple[object, ...] = () mtp_draft_mamba_pools: tuple[object, ...] = () + + @property + def swa_retain_window(self) -> Optional[int]: + # Cache history required by every reader of the SWA sub-pool. Retention, + # tree locks, prefix matching and host transfers must use the same span. + # The model's attention window remains sliding_window_size. + if self.sliding_window_size is None: + return None + return max(self.sliding_window_size, self.draft_swa_window) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index c4e26ea4d4..709be5b229 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -300,6 +300,7 @@ def build_kv_cache( hicache_draft_plan: Optional[HiCacheDraftPlan] = None, dynamo_kv_event_format: bool = False, kv_event_lora_names: Optional[KvEventLoraNames] = None, + draft_swa_window: int = 0, ) -> KVCacheBuildResult: parallel = get_parallel() sliding_window_size: Optional[int] = None @@ -324,6 +325,9 @@ def build_kv_cache( full_tokens_per_layer, swa_tokens_per_layer = ( tp_worker.get_tokens_per_layer_info() ) + assert draft_swa_window == 0 or sliding_window_size is not None, ( + "a fused draft reads an swa sub-pool this target does not build" + ) req_to_token_pool, token_to_kv_pool_allocator = tp_worker.get_memory_pool() mtp_draft_device_pools = tp_worker.model_runner.mtp_draft_device_pools @@ -398,9 +402,17 @@ def build_kv_cache( chunked_prefill_size=effective_chunked_prefill_size, sliding_window_size=sliding_window_size, swa_gates_prefix_match=swa_gates_prefix_match, + draft_swa_window=draft_swa_window, mtp_draft_device_pools=mtp_draft_device_pools, mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools, ) + if params.swa_retain_window != sliding_window_size: + logger.info( + "SWA eviction keeps %d tokens back: the fused draft reads past the " + "%d-token window.", + params.swa_retain_window, + sliding_window_size, + ) tree_context = TreeCacheBuildContext( server_args=server_args, diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 0a9eda5be7..ca58924f27 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -638,12 +638,8 @@ class KVCacheConfigurator: alloc = token_to_kv_pool_allocator placement = self._fused_draft_from_target_buffer(alloc) if placement is not None: - from sglang.srt.mem_cache.layout.fused_draft import ( - draft_swa_layer_ids, - ) from sglang.srt.mem_cache.unified_draft_pool import ( bind_fused_draft, - draft_kv_layer_ids, draft_state_layer_classes, ) @@ -651,27 +647,29 @@ class KVCacheConfigurator: req_to_token_pool = self._build_req_to_token_pool( max_num_reqs=sizes.max_running_requests ) + runner = self.draft_model_idx or 0 # The target placed this draft from its config; the built # model is the ground truth for whether it carries state. state_layers = draft_state_layer_classes(self.model) - if state_layers: + if state_layers and not placement.lanes_for(runner, "mamba"): raise ValueError( "Fused draft KV: the draft model has recurrent / " f"linear-attention layers ({', '.join(state_layers)}) " - "that the fused region gives no state pool." + "that the fused placement gives no state lanes." ) - draft_pool = bind_fused_draft( + binding = bind_fused_draft( + placement=placement, + runner=runner, unified_buffer=alloc.unified_buffer, host_allocator=alloc, - placement=placement, - runner=self.draft_model_idx or 0, - kv_layer_ids=draft_kv_layer_ids(self.model), - swa_layer_ids=draft_swa_layer_ids(self.model_config), + model=self.model, + model_config=self.model_config, + req_to_token_pool=req_to_token_pool, page_size=self.page_size, ) return _InitializedPools( - req_to_token_pool=req_to_token_pool, - token_to_kv_pool=draft_pool, + req_to_token_pool=binding.req_to_token_pool, + token_to_kv_pool=binding.token_to_kv_pool, token_to_kv_pool_allocator=alloc, unified_memory_pool=None, ) @@ -823,14 +821,20 @@ class KVCacheConfigurator: # The region holds rows in the target's KV dtype; a draft that resolved # its own would read and write them as something else. Compare the KV # dtypes, not their storage: every fp8 flavor is stored as uint8. - region_kv_dtype = placement.region.resolved_kv_dtype() - if self.kv_cache_dtype != region_kv_dtype: - raise ValueError( - f"Fused draft KV: the draft resolved its KV cache dtype to " - f"{self.kv_cache_dtype}, but its region inside the target's pages " - f"holds {region_kv_dtype}. Set " - "--speculative-draft-kv-cache-dtype to the target's KV cache dtype." - ) + from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS + + for host in placement.hosts(): + if HOST_KINDS[host].family != "dense": + continue + region_kv_dtype = placement.region(host).resolved_kv_dtype() + if self.kv_cache_dtype != region_kv_dtype: + raise ValueError( + f"Fused draft KV: the draft resolved its KV cache dtype to " + f"{self.kv_cache_dtype}, but its region inside the target's " + f"pages holds {region_kv_dtype}. Set " + "--speculative-draft-kv-cache-dtype to the target's KV cache " + "dtype." + ) return placement def _init_unified_mamba_pools( @@ -1038,6 +1042,15 @@ class KVCacheConfigurator: swa_v_head_dim=mc.swa_v_head_dim, ) + def _unified_host_names(self) -> tuple: + """The sub-pools the unified factory for this model builds, by name.""" + names = ["full"] + if self.is_hybrid_swa: + names.append("swa") + if self.mambaish_config is not None: + names.append("mamba") + return tuple(names) + def _fused_draft_decision(self): """Whether, and where, the draft's layers fuse into the target's sub-pools. An empty decision means fusion does not apply (unified @@ -1048,6 +1061,7 @@ class KVCacheConfigurator: to the kernel.""" from sglang.srt.mem_cache.layout.fused_draft import ( FusedDraftDecision, + PlacementContext, draft_kv_profile, place_fused_draft, ) @@ -1166,11 +1180,24 @@ class KVCacheConfigurator: return place_fused_draft( profile=profile, num_runners=num_runners, - asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(), + ctx=PlacementContext( + host_names=self._unified_host_names(), + target_window=self.model_config.sliding_window_size, + asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(), + draft_backends=self._draft_attention_backends(), + ), store_dtype=_store_dtype_for(self.kv_cache_dtype), kv_dtype=self.kv_cache_dtype, ) + def _draft_attention_backends(self) -> tuple: + """The backends the draft worker may run on: its explicit one, else + the target's, which it inherits.""" + explicit = get_spec().speculative_draft_attention_backend + if explicit is not None: + return (explicit,) + return tuple(sorted(b for b in attention_backends() if b is not None)) + def _draft_backends_carry_v_head_dim(self) -> bool: """Whether every resolved attention backend, the draft's included, carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The @@ -1187,10 +1214,69 @@ class KVCacheConfigurator: pad), built from the same spec as the pool factory's so the priced and allocated entries agree; None when the draft does not fuse into that sub-pool.""" + from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS + placement = self._fused_draft_decision().placement - if placement is None or sub_pool_name != "full": + if placement is None: return None - return self._full_host_spec(placement.region).entry_bytes() + region = placement.region(sub_pool_name) + if region is None: + return None + builders = { + "full": self._full_host_spec, + "swa": self._swa_host_spec, + "mamba": self._mamba_host_spec, + } + assert sub_pool_name in HOST_KINDS, ( + f"sub-pool {sub_pool_name!r} is not a registered host kind" + ) + assert sub_pool_name in builders, ( + f"no fused host spec builder for sub-pool {sub_pool_name!r}" + ) + return builders[sub_pool_name](region).entry_bytes() + + def _mamba_host_spec(self, region): + from sglang.srt.mem_cache.unified_memory_pool import MambaSubPoolSpec + + assert self.mambaish_config is not None, ( + "only a mamba-ish host builds a mamba sub-pool" + ) + cp = self.mambaish_config.mamba2_cache_params + # This runner's OWN state layers, as the mamba factory slices them. + layer_ids = [ + i + for i in cp.layers + if self.layer_info.start_layer <= i < self.layer_info.end_layer + ] + return MambaSubPoolSpec( + name="mamba", + layer_num=len(layer_ids), + conv_state_shapes=tuple(tuple(int(x) for x in s) for s in cp.shape.conv), + conv_dtype=cp.dtype.conv, + temporal_state_shape=tuple(int(x) for x in cp.shape.temporal), + temporal_dtype=cp.dtype.temporal, + grow_direction="up", + draft_region=region, + ) + + def _swa_host_spec(self, region): + from sglang.srt.mem_cache.unified_memory_pool import ( + MHASubPoolSpec, + _store_dtype_for, + ) + + assert self.is_hybrid_swa, "only a hybrid-SWA host builds an swa sub-pool" + geometry = self._unified_swa_head_geometry() + return MHASubPoolSpec( + name="swa", + layer_num=len(self.layer_info.swa_attention_layer_ids), + head_num=geometry.swa_head_num, + head_dim=geometry.swa_head_dim, + v_head_dim=geometry.swa_v_head_dim, + store_dtype=_store_dtype_for(self.kv_cache_dtype), + grow_direction="up", + draft_region=region, + ) def _full_host_spec(self, region): from sglang.srt.mem_cache.unified_memory_pool import ( @@ -1292,31 +1378,28 @@ class KVCacheConfigurator: def _fused_draft_for_pool_factory(self, decision): """The placement a pool factory is handed. It logs the decision it returns, so the boot log shows whether fusion engaged or why not.""" + from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS + if decision.placement is None: if decision.declined is not None: logger.warning("fused draft KV disabled: %s", decision.declined) return None placement = decision.placement - region = placement.region - kv_dtype = region.resolved_kv_dtype() - logger.info( - "[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d " - "kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s", - region.lane_num, - region.head_num, - region.head_dim, - region.resolved_v_head_dim(), - ( - kv_dtype - if kv_dtype == region.store_dtype - else f"{kv_dtype} (stored as {region.store_dtype})" - ), - region.entry_bytes(), - [ - tuple(placement.lanes_for(r)) - for r in range(len(placement.runner_lane_counts)) - ], - ) + if decision.note is not None: + logger.info( + "[unified-memory-pool] fused draft placement: %s", decision.note + ) + for host in placement.hosts(): + logger.info( + "[unified-memory-pool] %s", + HOST_KINDS[host].describe( + region=placement.region(host), + lanes=[ + tuple(placement.lanes_for(r, host)) + for r in range(len(placement.runners)) + ], + ), + ) return placement def _init_unified_swa_pools( @@ -2956,9 +3039,15 @@ class KVCacheConfigurator: else: max_stage_mamba_layers = len(all_mamba_layers) pp_layer_scale = max_stage_mamba_layers / max(len(all_mamba_layers), 1) - stage_per_req = int( - config.mamba2_cache_params.mamba_cache_per_req * pp_layer_scale + # A fused draft state block widens every state slot: price the entry + # the unified factory carves out (host + draft + pad), not the host's. + fused_state_entry = self.fused_entry_bytes("mamba") + per_req = ( + config.mamba2_cache_params.mamba_cache_per_req + if fused_state_entry is None + else fused_state_entry ) + stage_per_req = int(per_req * pp_layer_scale) has_spec_dec = not self.spec_algorithm.is_none() # ReplaySSM drops the per-step intermediate_ssm scratch, so its mamba budget diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 69b2109f91..e97fd75501 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -226,6 +226,19 @@ class KVIndexTranslator: None when its pool has no such sub-pool.""" return self._spaces.get(kind) + @property + def window_kv_kind(self) -> IdSpaceKind: + """Storage used by window attention, independent of its attention mask. + + A fused window draft may live in FULL when SWA cannot host its window. + Such a draft still clips its reads, but uses FULL physical addresses. + """ + return ( + IdSpaceKind.SLIDING_WINDOW + if IdSpaceKind.SLIDING_WINDOW in self._spaces + else IdSpaceKind.FULL + ) + def write_ids( self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL ) -> Optional[torch.Tensor]: @@ -406,7 +419,7 @@ class KVIndexTranslator: reads stay virtual (a static pool, or DCP).""" if not self._reads_translated(kind): return self._passthrough_table(req_pool_indices) - if plan.has_read_table(kind): + if plan.has_read_table(kind, reader=self): return self._plan_table(plan, kind=kind, rows=bs) return KVIndexTable( ids=self.req_to_token, @@ -444,7 +457,7 @@ class KVIndexTranslator: "a translating reader must read through the plan of its own " "req_to_token rows" ) - return plan.read_table(kind=kind, rows=rows, into=into) + return plan.read_table(kind=kind, rows=rows, into=into, reader=self) def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable: return KVIndexTable( @@ -474,7 +487,7 @@ class KVIndexTranslator: (a static pool, or DCP) for the caller to finish; a static SWA pool can fuse its full->swa table into the gather as ``token_mapping``.""" bs = int(seq_lens.numel()) - if self._reads_translated(kind) and not plan.has_read_table(kind): + if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self): assert plan.is_read_by(self), ( "a translating reader must read through the plan of its own " "req_to_token rows" @@ -534,7 +547,7 @@ class KVIndexTranslator: "DCP, where the caller selects this rank's share itself)" ) rows = out.shape[0] - first = not plan.has_read_table(kind) + first = not plan.has_read_table(kind, reader=self) table = self._plan_table( plan, kind=kind, rows=rows, into=out if first else None ) diff --git a/python/sglang/srt/mem_cache/kv_loc_plan.py b/python/sglang/srt/mem_cache/kv_loc_plan.py index 91da2707c6..1ecb9dc866 100644 --- a/python/sglang/srt/mem_cache/kv_loc_plan.py +++ b/python/sglang/srt/mem_cache/kv_loc_plan.py @@ -293,9 +293,14 @@ class KVLocPlan: # -- reads ----------------------------------------------------------------- - def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool: + def has_read_table( + self, + kind: IdSpaceKind = IdSpaceKind.FULL, + *, + reader: Optional[KVIndexTranslator] = None, + ) -> bool: """Whether a reader has had the ``kind`` table built yet.""" - return self._source.space(kind).key in self._read_tables + return self._read_space(kind, reader).key in self._read_tables def read_table( self, @@ -303,14 +308,17 @@ class KVLocPlan: kind: IdSpaceKind = IdSpaceKind.FULL, rows: Optional[int] = None, into: Optional[torch.Tensor] = None, + reader: Optional[KVIndexTranslator] = None, ) -> KVIndexTable: """The ``kind`` page table over ``[0, seq_lens + read_extent)``, built when a table reader first asks (in place in ``into``, that reader's capture-stable table, when given) and shared by every later reader. ``rows=n`` returns exactly n rows; lanes past the plan's batch read the sink, appended by copying, never by translating again. Where reads stay - virtual (a static pool, or DCP) it is the `req_to_token` passthrough.""" - space = self._source.space(kind) + virtual (a static pool, or DCP) it is the `req_to_token` passthrough. + ``reader`` names the sub-pool, as in `write_ids`; the plan's own + runner by default.""" + space = self._read_space(kind, reader) table = self._read_tables.get(space.key) if table is None or ( table.is_translated and rows is not None and table.ids.shape[0] < rows @@ -329,6 +337,13 @@ class KVLocPlan: ) return table + def _read_space( + self, kind: IdSpaceKind, reader: Optional[KVIndexTranslator] + ) -> IdSpace: + # A later depth of a multi-layer draft can route window layers that + # the depth whose runner built the plan does not. + return (self._source if reader is None else reader).space(kind) + def is_read_by(self, reader: KVIndexTranslator) -> bool: """Whether `reader` reads this plan's tables: it indexes the plan's physical ids and gathers the same `req_to_token` rows. A reader with a diff --git a/python/sglang/srt/mem_cache/layout/fused_draft.py b/python/sglang/srt/mem_cache/layout/fused_draft.py index ba2c830a72..9ec70dbb13 100644 --- a/python/sglang/srt/mem_cache/layout/fused_draft.py +++ b/python/sglang/srt/mem_cache/layout/fused_draft.py @@ -2,35 +2,53 @@ The nesting, outermost first: - sub-pool a named pool inside the unified pool, with its own grow direction - and frontier. HOST is the role it plays when its entries also - carry draft bytes; only "full" hosts a draft here. + sub-pool a named pool inside the unified pool ("full", "swa", "mamba"), + with its own grow direction and frontier. HOST is the role it + plays when its entries also carry draft bytes. slot one token entry of that sub-pool. The draft's KV is extra parts of every slot, after the host's own parts, so ONE slot id per token covers target and draft: one allocation, one free, one whole-page relocation carry both. - region the draft geometry inside the host's entries. A region is not a + region the draft geometry inside one host's entries. A region is not a `SubPoolSpec`: no grow direction, no frontier logic, only geometry. lane one position in the region's layer dimension -- one draft layer of one runner, present in every slot. + range the contiguous lanes one runner owns in one host. A lane is not a layer. Runners are the draft's execution copies: a replicated head gives EVERY runner all the draft's layers, on lanes of its own so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes), while a per-depth head (one block per predicted position) gives each runner -ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a -contiguous run of lanes, and the runs tile the region in runner order. +ONE depth (8 layers across 2 runners = 2 lanes). The runners' ranges tile +each region in runner order, with no gap and no overlap. -A draft whose layers need another sub-pool kind declines fusion as a whole. +A draft LAYER kind (`LAYER_*`) is what the draft checkpoint has; a HOST kind +(`HostKind`, keyed by sub-pool name) is what a sub-pool can carry in its +entries. `place_fused_draft` consults the `HOST_KINDS` registry, and the draft +worker binds each host through its family's binder +(`unified_draft_pool.DRAFT_BINDERS`). When a kind's host turns it away, its +layers FOLD into that host's `fallback_host` if the one-region-one-geometry +rule allows. """ -from typing import List, Optional, Tuple +import math +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union import msgspec import torch from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart +LAYER_FULL = "full" +LAYER_WINDOW = "window" +LAYER_STATE = "state" + +_LAYER_KIND_NOUN = { + LAYER_FULL: "full-attention", + LAYER_WINDOW: "sliding-window", + LAYER_STATE: "recurrent-state", +} + class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True): """Geometry of the DRAFT model's K/V rows fused into every slot of a host @@ -94,28 +112,6 @@ class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True): ) -class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True): - """Where every draft runner's layers live inside the "full" sub-pool: the - region, and each runner's lane count in runner order. Built once on the - target, stored on the `UnifiedKVPool`, and read back by each draft - runner, so the two sides cannot disagree on a lane. - """ - - region: DenseDraftRegion - runner_lane_counts: Tuple[int, ...] - - def __post_init__(self): - assert self.runner_lane_counts, "a placement needs at least one draft runner" - assert sum(self.runner_lane_counts) == self.region.lane_num, ( - f"runner lane counts {self.runner_lane_counts} must tile " - f"range({self.region.lane_num})" - ) - - def lanes_for(self, runner: int) -> range: - start = sum(self.runner_lane_counts[:runner]) - return range(start, start + self.runner_lane_counts[runner]) - - class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True): """Per-GPU K/V row geometry of one kind of draft attention layer.""" @@ -124,37 +120,345 @@ class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True): v_head_dim: int +class DraftStateGeometry(msgspec.Struct, frozen=True, kw_only=True): + """Per-GPU shapes of one recurrent-state layer of the draft: one conv + tensor per stream plus the temporal state, as `MambaPool.State` lays them + out. Every stream is carried, even ones a depth never touches, because + the backend indexes ``conv[stream]`` by absolute stream number.""" + + conv_state_shapes: Tuple[Tuple[int, ...], ...] + conv_dtype: torch.dtype + temporal_state_shape: Tuple[int, ...] + temporal_dtype: torch.dtype + + def conv_row_bytes(self, idx: int) -> int: + return math.prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize + + def temporal_row_bytes(self) -> int: + return math.prod(self.temporal_state_shape) * self.temporal_dtype.itemsize + + def layer_bytes(self) -> int: + conv = sum(self.conv_row_bytes(i) for i in range(len(self.conv_state_shapes))) + return conv + self.temporal_row_bytes() + + +class DraftStateRegion(msgspec.Struct, frozen=True, kw_only=True): + """The draft's recurrent state fused into every slot of a host state + sub-pool: one block per fused (runner, layer), laid out stream-major like + the host's own block, after it. The host's whole-entry clear and copy then + carry the draft's state with the target's.""" + + lane_num: int + state: DraftStateGeometry + + def validate(self) -> None: + assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}" + assert len(self.state.conv_state_shapes) > 0, ( + "conv_state_shapes must be non-empty" + ) + + def entry_bytes(self) -> int: + """Draft state bytes per slot, before the host entry's alignment.""" + return self.lane_num * self.state.layer_bytes() + + +DraftRegion = Union[DenseDraftRegion, DraftStateRegion] + + +class DraftLayerSet(msgspec.Struct, frozen=True, kw_only=True): + """The draft's layers of one kind: layer ids for a replicated head, depth + ids for a per-depth head, with the per-GPU geometry they share.""" + + layer_ids: Tuple[int, ...] + geometry: Any + # LAYER_WINDOW only: how many tokens back the layers read. + window: Optional[int] = None + + class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True): - """What the draft checkpoint asks of the host sub-pools. + """What the draft checkpoint asks of the host sub-pools, by layer kind. ``num_depths`` > 1 marks a per-depth head (one transformer block per MTP - depth, served by one runner each under multi-layer EAGLE); otherwise - every runner serves all ``num_layers`` layers. + depth, served by one runner each under multi-layer EAGLE), whose layer + ids are depth ids; otherwise every runner serves all ``num_layers`` + layers. A kind is present only when the draft has layers of it. """ num_layers: int - full: DraftKVGeometry - swa_layer_ids: Tuple[int, ...] = () num_depths: int = 1 - num_state_layers: int = 0 + kinds: Dict[str, DraftLayerSet] = {} -def draft_state_layer_num(draft_model_config) -> int: - """Recurrent-state layers the draft HEAD owns -- not the trunk's. +class PlacementContext(msgspec.Struct, frozen=True, kw_only=True): + """Target-side facts a host kind consults when placing a layer set: + the sub-pools the host builds, the target's window, whether every + resolved backend carries v_head_dim, and the backends the draft worker + may run on (its explicit one, else the target's).""" - A plain NEXTN head of a linear-attention trunk inherits the trunk's - config class, so `mamba2_cache_params` lists the TRUNK's state layers - while the head itself is a full-attention block. Only a conv-chain MTP - head declares `mtp_local_layer_ids`, so that is what separates them. + host_names: Tuple[str, ...] + target_window: Optional[int] = None + asymmetric_rows_ok: bool = False + draft_backends: Tuple[str, ...] = () + + +class HostKind(msgspec.Struct, frozen=True, kw_only=True): + """A unified sub-pool that carries one kind of draft layer in its entries. + + ``serves`` is the layer kind this host is the first choice for and + ``family`` picks the draft worker's binder. ``fallback_host`` carries + ``serves`` when this host cannot, under the one-region-one-geometry fold + rule. """ - from sglang.srt.configs.hybrid_arch import mambaish_config - mambaish = mambaish_config(draft_model_config) - if mambaish is None: - return 0 - if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None: - return 0 - return len(mambaish.mamba2_cache_params.layers) + name: str + serves: str + family: str + fallback_host: Optional[str] = None + + def prerequisite( + self, *, layers: DraftLayerSet, ctx: PlacementContext + ) -> Optional[str]: + """What the runtime must carry for ``layers`` to be served fused at + all, whichever host takes them; a reason here declines the draft.""" + return None + + def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]: + """None when this host's entries can carry ``layers``, else why not.""" + if self.name not in ctx.host_names: + return f"the host has no {self.name!r} sub-pool" + return None + + def region_for( + self, + *, + layers: DraftLayerSet, + lane_num: int, + store_dtype: torch.dtype, + kv_dtype: Optional[torch.dtype], + ) -> DraftRegion: + raise NotImplementedError + + def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str: + """The boot-log line for ``region`` and each runner's lanes in it.""" + raise NotImplementedError + + +class DenseHostKind(HostKind): + """A host whose entries take the draft's K/V rows as two more dense parts.""" + + def region_for( + self, + *, + layers: DraftLayerSet, + lane_num: int, + store_dtype: torch.dtype, + kv_dtype: Optional[torch.dtype], + ) -> DenseDraftRegion: + geometry = layers.geometry + assert isinstance(geometry, DraftKVGeometry), geometry + return DenseDraftRegion( + lane_num=lane_num, + head_num=geometry.head_num, + head_dim=geometry.head_dim, + v_head_dim=geometry.v_head_dim, + store_dtype=store_dtype, + kv_dtype=kv_dtype, + ) + + def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str: + kv_dtype = region.resolved_kv_dtype() + stored = ( + kv_dtype + if kv_dtype == region.store_dtype + else f"{kv_dtype} (stored as {region.store_dtype})" + ) + return ( + f"fused draft region in {self.name!r}: {region.lane_num} lane(s) x " + f"{region.head_num} kv head(s) x {region.head_dim}/" + f"{region.resolved_v_head_dim()} k/v head_dim @ {stored} " + f"= {region.entry_bytes()} B/token; runner lanes {list(lanes)}" + ) + + +# Registration order is the order placements report their hosts in. +HOST_KINDS: Dict[str, HostKind] = {} + + +def host_for(layer_kind: str) -> Optional[HostKind]: + """The registered host that serves ``layer_kind`` first, if any.""" + for kind in HOST_KINDS.values(): + if kind.serves == layer_kind: + return kind + return None + + +def register_host_kind(kind: HostKind) -> HostKind: + assert kind.name not in HOST_KINDS, f"host kind {kind.name!r} is already registered" + primary = host_for(kind.serves) + assert primary is None, ( + f"layer kind {kind.serves!r} is already served by host {primary.name!r}" + ) + assert kind.fallback_host is None or kind.fallback_host in HOST_KINDS, ( + f"host kind {kind.name!r} falls back to unregistered host " + f"{kind.fallback_host!r}" + ) + HOST_KINDS[kind.name] = kind + return kind + + +FULL_HOST = register_host_kind( + DenseHostKind(name="full", serves=LAYER_FULL, family="dense") +) + +# Multi-step draft backends that build the per-step sliding-window read and +# write rails a fused window layer needs (TritonMultiStepDraftBackend). +WINDOW_RAIL_BACKENDS = frozenset({"triton"}) + + +class WindowHostKind(DenseHostKind): + """The swa sub-pool: a draft window layer rides there when its window fits + the target's; eviction keeps any further reach the draft worker declares.""" + + def prerequisite( + self, *, layers: DraftLayerSet, ctx: PlacementContext + ) -> Optional[str]: + unrailed = sorted(set(ctx.draft_backends) - WINDOW_RAIL_BACKENDS) + if unrailed: + return ( + f"the draft's attention backend {unrailed} carries no per-step " + f"sliding-window rail (only {sorted(WINDOW_RAIL_BACKENDS)} does)" + ) + return None + + def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]: + reason = super().admits(layers=layers, ctx=ctx) + if reason is not None: + return reason + if layers.window is None: + return "the draft declares no sliding window size" + if ctx.target_window is None: + return "the target declares no sliding window size" + if layers.window > ctx.target_window: + return ( + f"its window {layers.window} exceeds the target's window " + f"{ctx.target_window}" + ) + return None + + +SWA_HOST = register_host_kind( + WindowHostKind( + name="swa", serves=LAYER_WINDOW, family="dense", fallback_host="full" + ) +) + + +class StateHostKind(HostKind): + """The mamba sub-pool: its entries take the draft's recurrent-state block + after the host's streams, so the slot that is the request's carries both.""" + + def region_for( + self, + *, + layers: DraftLayerSet, + lane_num: int, + store_dtype: torch.dtype, + kv_dtype: Optional[torch.dtype], + ) -> DraftStateRegion: + geometry = layers.geometry + assert isinstance(geometry, DraftStateGeometry), geometry + return DraftStateRegion(lane_num=lane_num, state=geometry) + + def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str: + state = region.state + return ( + f"fused draft state in {self.name!r}: {region.lane_num} lane(s) x " + f"{len(state.conv_state_shapes)} conv stream(s) @ {state.conv_dtype} + " + f"temporal {tuple(state.temporal_state_shape)} @ {state.temporal_dtype} " + f"= {region.entry_bytes()} B/slot; runner lanes {list(lanes)}" + ) + + +MAMBA_HOST = register_host_kind( + StateHostKind(name="mamba", serves=LAYER_STATE, family="state") +) + + +class RunnerLanes(msgspec.Struct, frozen=True, kw_only=True): + """One draft runner's lanes inside each host region: host -> (start, count).""" + + ranges: Dict[str, Tuple[int, int]] = {} + + def range_for(self, host: str) -> range: + start, count = self.ranges.get(host, (0, 0)) + return range(start, start + count) + + +class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True): + """Where every draft runner's layers live inside the host sub-pools. + + One region per host sub-pool that carries draft lanes plus each runner's + lane range into it. Built once on the target, stored on the + `UnifiedKVPool`, and read back by each draft runner, so the two sides + cannot disagree on a lane. + """ + + runners: Tuple[RunnerLanes, ...] + regions: Dict[str, DraftRegion] = {} + + def __post_init__(self): + assert len(self.runners) > 0, "a placement needs at least one draft runner" + hosts = set(self.regions).union(*(r.ranges for r in self.runners)) + for host in sorted(hosts): + assert host in HOST_KINDS, ( + f"no host kind is registered for sub-pool {host!r}" + ) + region = self.regions.get(host) + lanes = [s for r in self.runners for s in r.range_for(host)] + if region is None: + assert not lanes, f"host {host!r} has draft lanes but no region" + continue + assert lanes == list(range(region.lane_num)), ( + f"host {host!r}: runner lane ranges {lanes} must tile " + f"range({region.lane_num}) in runner order" + ) + + def region(self, host: str) -> Optional[DraftRegion]: + return self.regions.get(host) + + def hosts(self) -> Tuple[str, ...]: + """The hosts carrying draft lanes, in registry order.""" + return tuple(h for h in HOST_KINDS if h in self.regions) + + def lanes_for(self, runner: int, host: str) -> range: + return self.runners[runner].range_for(host) + + @classmethod + def from_counts( + cls, + *, + counts: Mapping[str, Sequence[int]], + regions: Mapping[str, DraftRegion], + ) -> "FusedDraftPlacement": + """Tile each host's region with the runners' layer counts, in runner order.""" + num_runners = {len(per_runner) for per_runner in counts.values()} + assert len(num_runners) == 1, ( + f"every host needs one layer count per runner; got {dict(counts)}" + ) + ranges: List[Dict[str, Tuple[int, int]]] = [ + {} for _ in range(num_runners.pop()) + ] + for host, per_runner in counts.items(): + assert host in regions, f"host {host!r} has layer counts but no region" + start = 0 + for runner, count in enumerate(per_runner): + if count: + ranges[runner][host] = (start, count) + start += count + return cls( + runners=tuple(RunnerLanes(ranges=r) for r in ranges), + regions=dict(regions), + ) def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]: @@ -167,6 +471,25 @@ def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]: return () +def draft_state_layer_ids(draft_model_config, *, num_depths: int) -> Tuple[int, ...]: + """The draft depth ids that own a recurrent-state block of their own. + + Only a per-depth conv-chain MTP head carries state: the baseline serves + it with a state cache cloned beside the target's. A plain NEXTN head of + a linear-attention trunk inherits the trunk's mamba-ish config class but + is a full-attention block; its config lists the TRUNK's state layers, + and the head carries none. + """ + from sglang.srt.configs.hybrid_arch import mambaish_config + + mc = draft_model_config + if mambaish_config(mc) is None or mc.num_nextn_predict_layers is None: + return () + if getattr(mc.hf_text_config, "mtp_local_layer_ids", None) is None: + return () + return tuple(range(num_depths)) + + def draft_kv_profile( draft_model_config, *, @@ -177,37 +500,76 @@ def draft_kv_profile( """The profile of a draft `ModelConfig`, heads divided by attn_tp the way the target divides its own (drafts never join the DCP group). ``num_depths`` overrides the config's `num_nextn_predict_layers`.""" + from sglang.srt.configs.hybrid_arch import mambaish_config + mc = draft_model_config if num_depths is None: num_depths = mc.num_nextn_predict_layers - return DraftKVProfile( - num_layers=int(num_layers), - full=DraftKVGeometry( - head_num=int(mc.get_num_kv_heads(attn_tp_size)), - head_dim=int(mc.head_dim), - v_head_dim=int(mc.v_head_dim), - ), - swa_layer_ids=draft_swa_layer_ids(mc), - num_depths=1 if num_depths is None else int(num_depths), - num_state_layers=draft_state_layer_num(mc), + num_depths = 1 if num_depths is None else int(num_depths) + num_layers = int(num_layers) + window_ids = draft_swa_layer_ids(mc) + assert set(window_ids) <= set(range(num_layers)), ( + f"draft window layer ids {window_ids} outside range({num_layers})" ) + kinds: Dict[str, DraftLayerSet] = {} + full_ids = tuple(i for i in range(num_layers) if i not in window_ids) + if full_ids: + kinds[LAYER_FULL] = DraftLayerSet( + layer_ids=full_ids, + geometry=DraftKVGeometry( + head_num=int(mc.get_num_kv_heads(attn_tp_size)), + head_dim=int(mc.head_dim), + v_head_dim=int(mc.v_head_dim), + ), + ) + if window_ids: + kinds[LAYER_WINDOW] = DraftLayerSet( + layer_ids=window_ids, + geometry=DraftKVGeometry( + head_num=int(mc.get_swa_num_kv_heads(attn_tp_size)), + head_dim=int(mc.swa_head_dim), + v_head_dim=int(mc.swa_v_head_dim), + ), + window=( + None if mc.sliding_window_size is None else int(mc.sliding_window_size) + ), + ) + state_ids = draft_state_layer_ids(mc, num_depths=num_depths) + if state_ids: + cp = mambaish_config(mc).mamba2_cache_params + kinds[LAYER_STATE] = DraftLayerSet( + layer_ids=state_ids, + geometry=DraftStateGeometry( + conv_state_shapes=tuple( + tuple(int(x) for x in s) for s in cp.shape.conv + ), + conv_dtype=cp.dtype.conv, + temporal_state_shape=tuple(int(x) for x in cp.shape.temporal), + temporal_dtype=cp.dtype.temporal, + ), + ) + return DraftKVProfile(num_layers=num_layers, num_depths=num_depths, kinds=kinds) class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True): """`place_fused_draft`'s answer: the placement, or why the draft cannot - fuse. Neither means fusion does not apply.""" + fuse. Neither means fusion does not apply. ``note`` says why a placed + layer kind did not get its first-choice host.""" placement: Optional[FusedDraftPlacement] = None declined: Optional[str] = None + note: Optional[str] = None def _runner_layer_counts( profile: DraftKVProfile, num_runners: int -) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]: - """Per runner, its (full, swa) layer counts; or why no runner layout exists.""" +) -> Tuple[Optional[Dict[str, List[int]]], Optional[str]]: + """Per layer kind, each runner's layer count; or why no runner layout exists.""" if profile.num_depths <= 1: - num_swa = len(profile.swa_layer_ids) - return [(profile.num_layers - num_swa, num_swa)] * num_runners, None + return { + kind: [len(layers.layer_ids)] * num_runners + for kind, layers in profile.kinds.items() + }, None if num_runners == 1: return None, ( f"a per-depth draft head ({profile.num_depths} depths) needs one " @@ -217,69 +579,128 @@ def _runner_layer_counts( return None, ( f"{num_runners} draft runners exceed the head's {profile.num_depths} depths" ) - swa = set(profile.swa_layer_ids) - return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None + return { + kind: [1 if r in layers.layer_ids else 0 for r in range(num_runners)] + for kind, layers in profile.kinds.items() + }, None + + +def _kv_rows_declined( + geometry: Any, *, store_dtype: torch.dtype, asymmetric_rows_ok: bool +) -> Optional[str]: + """Why one kind's K/V rows cannot be fused entry parts, or None.""" + if not isinstance(geometry, DraftKVGeometry): + return None + if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok: + return ( + "the draft's K/V rows are asymmetric " + f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) " + "and a resolved attention backend does not carry v_head_dim " + "through to the kernel" + ) + for dim in (geometry.head_dim, geometry.v_head_dim): + row_bytes = geometry.head_num * dim * store_dtype.itemsize + if row_bytes % ROW_ALIGN_BYTES: + return ( + f"the draft's K/V rows are {row_bytes} B, not a multiple of " + f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs" + ) + return None + + +def _fold_refused( + profile: DraftKVProfile, *, kind: str, into: HostKind +) -> Optional[str]: + """One region holds one row geometry: ``kind`` folds into ``into`` only + when the draft has no layers of the kind ``into`` serves, or both kinds + share a geometry.""" + own = profile.kinds.get(into.serves) + if own is None or own.geometry == profile.kinds[kind].geometry: + return None + return ( + f"its {_LAYER_KIND_NOUN[kind]} layers' rows differ from its " + f"{_LAYER_KIND_NOUN[into.serves]} layers' rows, so they cannot share the " + f"{into.name!r} sub-pool" + ) def place_fused_draft( *, profile: DraftKVProfile, num_runners: int, + ctx: PlacementContext, store_dtype: torch.dtype, - asymmetric_rows_ok: bool, kv_dtype: Optional[torch.dtype] = None, ) -> FusedDraftDecision: - """Assign every draft layer of every runner to the host sub-pool whose - lifetime covers what the layer reads: a full-attention layer rides in - ``"full"``. A layer kind no host sub-pool serves declines the whole draft, - as do asymmetric K/V rows unless the caller vouches that every attention - backend carries v_head_dim through to the kernel.""" + """Assign every draft layer of every runner to a host sub-pool through the + registry: a layer kind goes to the host that serves it, else to that + host's fallback under the fold rule. A kind no host takes declines the + whole draft, as does a kind whose runtime prerequisite is missing, + asymmetric K/V rows unless ``ctx`` vouches that every attention backend + carries v_head_dim through to the kernel, and rows off the entry-part + alignment.""" counts, reason = _runner_layer_counts(profile, num_runners) if counts is None: return FusedDraftDecision(declined=reason) - if profile.num_state_layers: - return FusedDraftDecision( - declined=( - f"the draft has {profile.num_state_layers} recurrent-state " - "layer(s) of its own, which no host state pool carries" - ) + for kind, layers in profile.kinds.items(): + what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)" + host = host_for(kind) + if host is None: + return FusedDraftDecision(declined=f"{what}: no host kind serves them") + reason = host.prerequisite(layers=layers, ctx=ctx) + if reason is not None: + return FusedDraftDecision(declined=f"{what}: {reason}") + reason = _kv_rows_declined( + layers.geometry, + store_dtype=store_dtype, + asymmetric_rows_ok=ctx.asymmetric_rows_ok, ) - num_swa = sum(swa for _, swa in counts) - if num_swa: - return FusedDraftDecision( - declined=( - f"the draft has {num_swa} SWA layer(s) of its own, which the " - "fused dense pool cannot serve" + if reason is not None: + return FusedDraftDecision(declined=reason) + + host_counts: Dict[str, List[int]] = {} + host_layers: Dict[str, DraftLayerSet] = {} + notes: List[str] = [] + for kind, layers in profile.kinds.items(): + host = host_for(kind) + what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)" + reason = host.admits(layers=layers, ctx=ctx) + if reason is not None: + fallback = ( + None if host.fallback_host is None else HOST_KINDS[host.fallback_host] ) - ) - geometry = profile.full - if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok: - return FusedDraftDecision( - declined=( - "the draft's K/V rows are asymmetric " - f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) " - "and a resolved attention backend does not carry v_head_dim " - "through to the kernel" - ) - ) - for dim in (geometry.head_dim, geometry.v_head_dim): - row_bytes = geometry.head_num * dim * store_dtype.itemsize - if row_bytes % ROW_ALIGN_BYTES: - return FusedDraftDecision( - declined=( - f"the draft's K/V rows are {row_bytes} B, not a multiple of " - f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs" + fold = None + if fallback is not None: + fold = _fold_refused(profile, kind=kind, into=fallback) + if fold is None: + fold = fallback.admits(layers=layers, ctx=ctx) + if fallback is None or fold is not None: + if fold is not None: + reason = f"{reason}, and {fold}" + return FusedDraftDecision( + declined=( + f"{what} cannot ride in the {host.name!r} sub-pool: {reason}" + ) ) - ) - full_counts = tuple(full for full, _ in counts) - region = DenseDraftRegion( - lane_num=sum(full_counts), - head_num=geometry.head_num, - head_dim=geometry.head_dim, - v_head_dim=geometry.v_head_dim, - store_dtype=store_dtype, - kv_dtype=kv_dtype, - ) + notes.append(f"{what} ride in the {fallback.name!r} sub-pool: {reason}") + host = fallback + per_runner = host_counts.setdefault(host.name, [0] * num_runners) + for runner, count in enumerate(counts[kind]): + per_runner[runner] += count + # The fold rule made every kind sharing a host share its geometry. + host_layers.setdefault(host.name, layers) + + placed = {host: c for host, c in host_counts.items() if sum(c) > 0} + regions = { + host: HOST_KINDS[host].region_for( + layers=host_layers[host], + lane_num=sum(c), + store_dtype=store_dtype, + kv_dtype=kv_dtype, + ) + for host, c in placed.items() + } return FusedDraftDecision( - placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts) + placement=FusedDraftPlacement.from_counts(counts=placed, regions=regions), + note="; ".join(notes) or None, ) diff --git a/python/sglang/srt/mem_cache/layout/token_major.py b/python/sglang/srt/mem_cache/layout/token_major.py index 351e864436..cce0cf55f3 100644 --- a/python/sglang/srt/mem_cache/layout/token_major.py +++ b/python/sglang/srt/mem_cache/layout/token_major.py @@ -13,7 +13,7 @@ so every per-layer view is ``(num_slots, *row_shape)`` with slot stride is the pool region's offset inside the raw buffer. """ -from typing import List, Sequence, Tuple +from typing import List, Optional, Sequence, Tuple import msgspec import torch @@ -179,21 +179,26 @@ def build_mamba_entry_views( temporal_dtype: torch.dtype, max_slots: int, anchor_bytes: int = 0, + entry_bytes: Optional[int] = None, + offset_bytes: int = 0, ) -> Tuple[List[torch.Tensor], torch.Tensor]: """Per-slot envelope views over ``raw`` for Mamba state. - Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]... - [temporal rows × layers]``. Each returned view has shape - ``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]`` - / ``.temporal``. Mamba state is always token-granular (page_size == 1). + Layout of one block: ``[conv[0] rows × layers][conv[1] rows × layers]... + [temporal rows × layers]``, starting ``offset_bytes`` into a slot's entry + of ``entry_bytes`` (the block's own size when omitted, i.e. a standalone + envelope). Each returned view has shape ``(num_layers, max_slots, + *inner_shape)`` matching ``MambaPool.State.conv[i]`` / ``.temporal``. + Mamba state is always token-granular (page_size == 1). """ - entry_bytes = mamba_entry_bytes( - layer_num=layer_num, - conv_state_shapes=conv_state_shapes, - conv_dtype=conv_dtype, - temporal_state_shape=temporal_state_shape, - temporal_dtype=temporal_dtype, - ) + if entry_bytes is None: + entry_bytes = mamba_entry_bytes( + layer_num=layer_num, + conv_state_shapes=conv_state_shapes, + conv_dtype=conv_dtype, + temporal_state_shape=temporal_state_shape, + temporal_dtype=temporal_dtype, + ) conv_itemsize = conv_dtype.itemsize assert entry_bytes % conv_itemsize == 0, ( @@ -207,7 +212,7 @@ def build_mamba_entry_views( as_conv_dtype = raw.view(conv_dtype) conv_slot_stride_elems = entry_bytes // conv_itemsize - offset_bytes_within_entry = 0 + offset_bytes_within_entry = offset_bytes conv_views: List[torch.Tensor] = [] for shape in conv_state_shapes: inner_shape_bytes = _prod(shape) * conv_itemsize diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 5cc72238dc..9568abec7c 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1971,6 +1971,8 @@ class KVWriteLoc: # swa_loc / full_loc are resolved once at metadata-init from the full # (padded) out_cache_loc; piecewise/DP-padded paths later narrow loc per # layer, so slice these pre-resolved locs to match (same per-token order). + # A multi-step draft step's slice is NOT a prefix of the all-steps loc; + # its backend derives per-step locs and never relies on this. if self.swa_loc is not None and self.swa_loc.shape[0] != self.loc.shape[0]: self.swa_loc = self.swa_loc[: self.loc.shape[0]] if self.full_loc is not None and self.full_loc.shape[0] != self.loc.shape[0]: diff --git a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py index 5b671ac2dd..01455c8504 100644 --- a/python/sglang/srt/mem_cache/pure_swa_radix_cache.py +++ b/python/sglang/srt/mem_cache/pure_swa_radix_cache.py @@ -23,6 +23,7 @@ class PureSWARadixCache(RadixCache): def __init__(self, params: CacheInitParams): super().__init__(params) self.sliding_window_size = params.sliding_window_size + self.draft_swa_window = params.draft_swa_window def supports_swa(self) -> bool: assert self.sliding_window_size is not None, ( diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 8902700587..2f44ea4486 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -468,7 +468,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): enable_hicache=False, write_through_threshold=256, device=str(self.device), - swa_sliding_window_size=params.sliding_window_size, + # Rust's tree window has the same cache-history contract as + # SWAComponent, including the fused draft's front rows. + swa_sliding_window_size=params.swa_retain_window, swa_req_ring=not params.swa_gates_prefix_match, enable_kv_cache_events=params.enable_kv_cache_events, dynamo_kv_event_format=params.dynamo_kv_event_format, diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index 21681cdb0d..20ae99f2cf 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -216,11 +216,12 @@ class SWAKVPool(BaseSWAKVPool): def get_v_head_dim(self): # The FULL side's dim, as HybridLinearKVPool.get_v_head_dim(): a caller # asking a pool for "the" v_head_dim wants the full-attention geometry. + # A window-only pool (a sliding-window MTP depth) has no full side; + # its swa side is the only geometry there is. # `start_layer`, not 0, so pipeline parallelism (start_layer > 0) works, # and because layer 0 need not be a full-attention layer. - return self.full_kv_pool.get_value_buffer(self.full_kv_pool.start_layer).shape[ - -1 - ] + lead = self.full_kv_pool if self.full_layer_nums > 0 else self.swa_kv_pool + return lead.get_value_buffer(lead.start_layer).shape[-1] def get_kv_buffer(self, layer_id: int): self._wait_for_layer(layer_id) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index f835bba12a..96176b8090 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -88,7 +88,9 @@ class SWAComponent(TreeComponent): super().__init__(cache, params) self._session_leaf_covered_len: dict[str, dict[UnifiedTreeNode, int]] = {} - self.sliding_window_size = params.sliding_window_size + # This is the tree's required history, including widened draft reads; + # it does not change the model's attention window. + self.sliding_window_size = params.swa_retain_window self.gates_prefix_match = params.swa_gates_prefix_match self.full_window_pages = ( self.sliding_window_size + params.page_size - 1 diff --git a/python/sglang/srt/mem_cache/unified_draft_pool.py b/python/sglang/srt/mem_cache/unified_draft_pool.py index 313a4c38d5..8f02e4b11a 100644 --- a/python/sglang/srt/mem_cache/unified_draft_pool.py +++ b/python/sglang/srt/mem_cache/unified_draft_pool.py @@ -1,14 +1,37 @@ """KV pools a DRAFT runner binds over the draft lanes fused into the target's unified pool: same pages, same slot ids, same v2p table as the target -- one -allocation, one free, one relocation.""" +allocation, one free, one relocation. -from typing import Any, Dict, List, Mapping, Optional, Sequence +`DRAFT_BINDERS` maps a `HostKind.family` to the binder that builds the +runner's pool over every host of that family; registering a binder is how a +new host kind reaches the draft worker. +""" +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple + +import msgspec import torch -from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement -from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool -from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool +from sglang.srt.mem_cache.layout.fused_draft import ( + HOST_KINDS, + DraftStateRegion, + FusedDraftPlacement, + draft_state_layer_ids, + draft_swa_layer_ids, +) +from sglang.srt.mem_cache.memory_pool import ( + KVWriteLoc, + MambaPool, + MHATokenToKVPool, + unwrap_write_loc, + write_loc_is_physical, +) +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool +from sglang.srt.mem_cache.unified_memory_pool import ( + UnifiedHybridReqToTokenPool, + UnifiedKVPool, +) +from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter class UnifiedDraftKVPool(MHATokenToKVPool): @@ -102,10 +125,314 @@ class UnifiedDraftKVPool(MHATokenToKVPool): raise NotImplementedError("CPU offloading is unsupported for fused draft KV.") +class UnifiedDraftSWAKVPool(SWAKVPool): + """A draft's KV pool when it has sliding-window layers: one dense draft + pool per host sub-pool it fuses into ("full" and/or "swa"), routed per + layer exactly like the target's `UnifiedSWAKVPool`, so the attention + backends run their swa rail for it. Inherits `SWAKVPool` for `isinstance` + only; never calls its `__init__` (it would allocate static pools).""" + + requires_physical_write_loc = True + + def __init__( + self, + *, + unified_buffer: UnifiedKVPool, + host_allocator, + page_size: int, + full_layer_lanes: Mapping[int, int], + swa_layer_lanes: Mapping[int, int], + ): + assert swa_layer_lanes, ( + "UnifiedDraftSWAKVPool binds at least one window layer; a full-only " + "draft binds UnifiedDraftKVPool" + ) + self.unified_buffer = unified_buffer + self.host_allocator = host_allocator + self.page_size = page_size + self.device = unified_buffer.device + self.layer_transfer_counter = None + self.full_layer_nums = len(full_layer_lanes) + self.swa_layer_nums = len(swa_layer_lanes) + self.layer_num = self.full_layer_nums + self.swa_layer_nums + self.start_layer = min([*full_layer_lanes, *swa_layer_lanes]) + self.size = unified_buffer.max_slots("full") - 1 + self.size_swa = unified_buffer.max_slots("swa") - 1 + + self.full_kv_pool: Optional[UnifiedDraftKVPool] = None + if full_layer_lanes: + self.full_kv_pool = self._side_pool("full", full_layer_lanes) + self.swa_kv_pool = self._side_pool("swa", swa_layer_lanes) + lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool + self.dtype = lead.dtype + self.head_num = lead.head_num + self.head_dim = lead.head_dim + + # {layer_id: (per-side index, is_swa_layer)}, sides indexed in layer order. + self.layers_mapping: Dict[int, Tuple[int, bool]] = {} + for idx, layer_id in enumerate(sorted(full_layer_lanes)): + self.layers_mapping[layer_id] = (idx, False) + for idx, layer_id in enumerate(sorted(swa_layer_lanes)): + self.layers_mapping[layer_id] = (idx, True) + # None so dispatch routes through the host's v2p tables, never a + # registered mapping. + self.full_to_swa_index_mapping: Optional[torch.Tensor] = None + self.enable_custom_mem_pool = False + self.custom_mem_pool = None + self.dsa_kv_cache_store_fp8 = False + self.kv_cache_dim = None + self.index_head_dim = None + self.mem_usage = 0.0 # fused into the host entries + + def _side_pool( + self, host_sub_pool_name: str, layer_lanes: Mapping[int, int] + ) -> UnifiedDraftKVPool: + # The side is indexed 0..n-1 in layer order (`layer_id_override`). + return UnifiedDraftKVPool( + unified_buffer=self.unified_buffer, + host_sub_pool_name=host_sub_pool_name, + host_allocator=self.host_allocator, + layer_lanes={ + idx: layer_lanes[layer_id] + for idx, layer_id in enumerate(sorted(layer_lanes)) + }, + page_size=self.page_size, + ) + + def _side(self, layer_id: int) -> Tuple[UnifiedDraftKVPool, int]: + pool_layer_id, is_swa = self.layers_mapping[layer_id] + pool = self.swa_kv_pool if is_swa else self.full_kv_pool + assert pool is not None, layer_id + return pool, pool_layer_id + + # -- KVCache surface a side of None cannot answer -- + + @property + def post_capture_active(self) -> bool: + return False # the host buffer is fully backed at boot + + @property + def post_capture_backed_bytes(self) -> int: + return 0 + + def finalize_backing(self, config) -> None: + return + + def register_layer_transfer_counter(self, layer_transfer_counter): + self.layer_transfer_counter = layer_transfer_counter + + # -- BaseSWAKVPool ABC surface -- + + def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: + return # the host's swa v2p table IS the mapping + + def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor): + """Virtual token ids -> swa-physical token ids, through the host.""" + return self.host_allocator.translate_loc_from_full_to_swa(kv_indices) + + def get_state_buf_infos(self): + raise NotImplementedError( + "fused draft KV has no per-layer contiguous regions; KV transfer / " + "disaggregation is unsupported." + ) + + # -- size/info -- + + def get_kv_size_bytes(self): + return 0, 0 # fused into the host entries; UnifiedKVPool logs the total + + def get_contiguous_buf_infos(self): + raise NotImplementedError( + "fused draft KV has no per-layer contiguous regions; KV transfer / " + "disaggregation is unsupported." + ) + + def get_v_head_dim(self): + lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool + return lead.get_value_buffer(lead.start_layer).shape[-1] + + # -- buffer accessors -- + + def get_key_buffer(self, layer_id: int): + pool, pool_layer_id = self._side(layer_id) + return pool.get_key_buffer(pool_layer_id) + + def get_value_buffer(self, layer_id: int): + pool, pool_layer_id = self._side(layer_id) + return pool.get_value_buffer(pool_layer_id) + + def get_kv_buffer(self, layer_id: int): + pool, pool_layer_id = self._side(layer_id) + return pool.get_kv_buffer(pool_layer_id) + + # -- kv writing -- + + def set_kv_buffer( + self, + layer, + loc_info, + cache_k: torch.Tensor, + cache_v: torch.Tensor, + k_scale: float = 1.0, + v_scale: float = 1.0, + ): + """Route to the right side. Both locs are physical already (the + backend derives `swa_loc` once per forward); never translates here. + The full side writes through `loc`: the capture-stable `full_loc` + alias is sized for the target's writes, not a per-step draft slice.""" + loc, swa_loc, _ = unwrap_write_loc(loc_info) + physical = write_loc_is_physical(loc_info) + pool, pool_layer_id = self._side(layer.layer_id) + if pool is self.swa_kv_pool: + assert swa_loc is not None, ( + "UnifiedDraftSWAKVPool.set_kv_buffer: window layer received no " + "swa_loc; the attention backend must bundle " + "forward_metadata.swa_out_cache_loc." + ) + side_loc = swa_loc + else: + side_loc = loc + pool.set_kv_buffer( + None, + KVWriteLoc(side_loc, physical=physical), + cache_k, + cache_v, + k_scale, + v_scale, + layer_id_override=pool_layer_id, + ) + + def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor): + raise NotImplementedError( + "fused draft KV relocates with the HOST pool's whole-page move; a " + "draft-side per-slot move would corrupt the fused layout" + ) + + def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): + raise NotImplementedError("CPU offloading is unsupported for fused draft KV.") + + def load_cpu_copy( + self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None + ): + raise NotImplementedError("CPU offloading is unsupported for fused draft KV.") + + +class UnifiedDraftMambaPool(MambaPool): + """A draft runner's recurrent state as views of the draft block fused into + the host's state entries: `mamba_cache.conv[stream][layer]` is laid out + exactly as a private `MambaPool`'s. Slot ids are the target's, translated + by the shared allocator. Pure view: the host's whole-entry clear and copy + already cover the draft block, so the block ops here act on the draft block alone, and it + never calls `MambaPool.__init__` (no buffers of its own).""" + + def __init__( + self, + *, + unified_buffer: UnifiedKVPool, + sub_pool_name: str, + layer_lanes: Mapping[int, int], + ): + spec = unified_buffer.mamba_spec(sub_pool_name) + region = spec.draft_region + assert isinstance(region, DraftStateRegion), ( + f"UnifiedDraftMambaPool: sub-pool {sub_pool_name!r} carries no fused " + "draft state region" + ) + layer_ids = sorted(layer_lanes) + assert layer_ids, "UnifiedDraftMambaPool binds at least one layer" + lanes = [layer_lanes[layer_id] for layer_id in layer_ids] + # A runner's lanes are one contiguous range, so its layers are one + # strided slice of the region's views (a gather would copy). + start = lanes[0] + assert lanes == list(range(start, start + len(lanes))) and ( + 0 <= start and start + len(lanes) <= region.lane_num + ), ( + f"draft state lanes {lanes} must be a contiguous range within " + f"{region.lane_num}" + ) + conv_views, temporal_view = unified_buffer.build_draft_state_views( + sub_pool_name + ) + stop = start + len(lanes) + + self._unified_buffer = unified_buffer + self._sub_pool_name = sub_pool_name + self.layer_lanes: Dict[int, int] = dict(layer_lanes) + self.mamba_layer_ids = layer_ids + self.num_mamba_layers = len(layer_ids) + self._max_size = unified_buffer.max_slots(sub_pool_name) - 1 + self.size = self._max_size + self.device = unified_buffer.device + self.memory_saver_adapter = TorchMemorySaverAdapter.create(enable=False) + self.enable_custom_mem_pool = False + self.custom_mem_pool = None + # Same disabled-state attributes `UnifiedMambaPool` replicates for the + # base class's unconditional reads. + self.enable_linear_replayssm = False + self.linear_replayssm_cache_len = 16 + self.replayssm_write_pos = None + self.replayssm_is_kda = False + self.enable_linear_replayssm_spec = False + self.replayssm_spec_fold = False + self.replayssm_cache_base = None + self.replayssm_is_flush = None + self.debug_memory_pool = False + self.conv_shard_groups = None + self.conv_slice_axis = spec.conv_slice_axis + self.mamba_cache = self.State( + conv=[view[start:stop] for view in conv_views], + temporal=temporal_view[start:stop], + ) + self.mem_usage = 0.0 # fused into the host entries + + def clear_slots(self, indices: torch.Tensor): + """Zero THIS runner's state block in the given slots. + + The block only, never the whole entry: the host's own streams share + the entry and must survive, since a new request takes over the + draft's block while the target's state for that slot is still live. + The host's whole-entry clear covers the block too, but only on a + plain EXTEND forward: ModelRunner skips it on decode, target-verify + and draft-extend, the modes the draft needs it in. + """ + for view in self.mamba_cache.conv: + view[:, indices] = 0 + self.mamba_cache.temporal[:, indices] = 0 + + def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor): + """Copy-on-write THIS runner's state block, block only (see above).""" + for view in self.mamba_cache.conv: + view[:, dst_indices] = view[:, src_indices] + self.mamba_cache.temporal[:, dst_indices] = self.mamba_cache.temporal[ + :, src_indices + ] + + def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor): + raise NotImplementedError( + "fused draft state relocates with the HOST pool's whole-entry move" + ) + + def get_contiguous_buf_infos(self): + raise NotImplementedError( + "fused draft state has no contiguous region of its own; state " + "transfer / disaggregation is unsupported." + ) + + def get_cpu_copy(self, indices): + raise NotImplementedError( + "CPU offloading is unsupported for fused draft state." + ) + + def load_cpu_copy(self, mamba_cache_cpu, indices): + raise NotImplementedError( + "CPU offloading is unsupported for fused draft state." + ) + + def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]: """The host allocator a fused draft pool translates through, or None for any other pool.""" - if isinstance(token_to_kv_pool, UnifiedDraftKVPool): + if isinstance(token_to_kv_pool, (UnifiedDraftKVPool, UnifiedDraftSWAKVPool)): return token_to_kv_pool.host_allocator return None @@ -123,8 +450,8 @@ def draft_kv_layer_ids(model) -> List[int]: def draft_state_layer_classes(model) -> List[str]: """Class names of the BUILT draft model's recurrent / linear-attention - modules. A fused draft has KV lanes only, so any of these would run with - no state pool of its own.""" + modules. A runner the placement gives no state lanes would run any of + these with no state pool of its own.""" from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 from sglang.srt.layers.radix_linear_attention import RadixLinearAttention @@ -137,37 +464,170 @@ def draft_state_layer_classes(model) -> List[str]: ) +class FusedDraftBinding(msgspec.Struct, frozen=True, kw_only=True): + """What a draft runner binds: its KV pool over the fused lanes and the + request table it reads.""" + + token_to_kv_pool: Any = None + req_to_token_pool: Any = None + + +class BindContext(msgspec.Struct, frozen=True, kw_only=True): + """One runner's inputs to every binder.""" + + placement: FusedDraftPlacement + runner: int + unified_buffer: Any + host_allocator: Any + model: Any + model_config: Any + page_size: int + + def hosts_of(self, family: str) -> Tuple[str, ...]: + """This runner's hosts of ``family`` that hold lanes for it.""" + return tuple( + host + for host in self.placement.hosts() + if HOST_KINDS[host].family == family + and len(self.placement.lanes_for(self.runner, host)) > 0 + ) + + +Binder = Callable[[BindContext, FusedDraftBinding], FusedDraftBinding] + +DRAFT_BINDERS: Dict[str, Binder] = {} + + +def register_draft_binder(family: str, binder: Binder) -> Binder: + assert family not in DRAFT_BINDERS, ( + f"a draft binder is already registered for host family {family!r}" + ) + DRAFT_BINDERS[family] = binder + return binder + + def bind_fused_draft( *, - unified_buffer: UnifiedKVPool, - host_allocator, placement: FusedDraftPlacement, runner: int, - kv_layer_ids: Sequence[int], - swa_layer_ids: Sequence[int], + unified_buffer: UnifiedKVPool, + host_allocator, + model, + model_config, + req_to_token_pool, page_size: int, -) -> UnifiedDraftKVPool: - """The KV pool draft runner ``runner`` binds over its fused slots. - - The placement sized the lanes from the draft config; the model's real - layer ids fill them in layer order, so a count mismatch is a loud boot - failure, never a silent alias. - """ - swa = set(swa_layer_ids) - full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa] - assert len(full_ids) == len(kv_layer_ids), ( - f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused " - "SWA KV is not supported yet" - ) - full_lanes = placement.lanes_for(runner) - assert len(full_ids) == len(full_lanes), ( - f"draft runner {runner}: {len(full_ids)} full-attention layer(s) " - f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}" - ) - return UnifiedDraftKVPool( +) -> FusedDraftBinding: + """Bind draft runner ``runner`` to its fused lanes: one binder per host + family that holds lanes for it, in registry order, each refining the + binding the previous one returned.""" + ctx = BindContext( + placement=placement, + runner=runner, unified_buffer=unified_buffer, - host_sub_pool_name="full", host_allocator=host_allocator, - layer_lanes=dict(zip(full_ids, full_lanes)), + model=model, + model_config=model_config, page_size=page_size, ) + families: List[str] = [] + for host in placement.hosts(): + family = HOST_KINDS[host].family + if family not in families and len(placement.lanes_for(runner, host)) > 0: + families.append(family) + assert families, f"draft runner {runner} holds no lane in any host" + binding = FusedDraftBinding(req_to_token_pool=req_to_token_pool) + for family in families: + assert family in DRAFT_BINDERS, ( + f"no draft binder is registered for host family {family!r}" + ) + binding = DRAFT_BINDERS[family](ctx, binding) + assert binding.token_to_kv_pool is not None, ( + f"draft runner {runner}: no binder produced a KV pool" + ) + return binding + + +def _bind_dense(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding: + """This runner's attention layers, in layer order, over its lanes in the + dense hosts. The placement sized the lanes from the draft config; the + model's real layer ids fill them, so a count mismatch is a loud boot + failure, never a silent alias. Window layers follow the placement: they + bind the swa sub-pool when it holds a region for them, else the full + sub-pool, where a full lifetime covers any window.""" + hosts = ctx.hosts_of("dense") + assert set(hosts) <= {"full", "swa"}, f"unknown dense host(s) in {hosts}" + kv_layer_ids = draft_kv_layer_ids(ctx.model) + full_lanes = ctx.placement.lanes_for(ctx.runner, "full") + swa_lanes = ctx.placement.lanes_for(ctx.runner, "swa") + if not swa_lanes: + full_ids, swa_ids = list(kv_layer_ids), [] + elif not full_lanes: + full_ids, swa_ids = [], list(kv_layer_ids) + else: + # Both kinds in one runner: a replicated head, whose config lists every + # window layer (a per-depth runner's config is clipped to its block). + window = set(draft_swa_layer_ids(ctx.model_config)) + full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in window] + swa_ids = [layer_id for layer_id in kv_layer_ids if layer_id in window] + assert len(full_ids) == len(full_lanes) and len(swa_ids) == len(swa_lanes), ( + f"draft runner {ctx.runner}: layers full={full_ids} swa={swa_ids} vs placed " + f"lanes full={list(full_lanes)} swa={list(swa_lanes)}" + ) + if not swa_ids: + pool = UnifiedDraftKVPool( + unified_buffer=ctx.unified_buffer, + host_sub_pool_name="full", + host_allocator=ctx.host_allocator, + layer_lanes=dict(zip(full_ids, full_lanes)), + page_size=ctx.page_size, + ) + else: + pool = UnifiedDraftSWAKVPool( + unified_buffer=ctx.unified_buffer, + host_allocator=ctx.host_allocator, + page_size=ctx.page_size, + full_layer_lanes=dict(zip(full_ids, full_lanes)), + swa_layer_lanes=dict(zip(swa_ids, swa_lanes)), + ) + return msgspec.structs.replace(binding, token_to_kv_pool=pool) + + +register_draft_binder("dense", _bind_dense) + + +def _runner_state_layer_ids(ctx: BindContext) -> Tuple[int, ...]: + """The recurrent-state layer ids this runner serves: its own depth for a + per-depth head, every state layer otherwise.""" + mc = ctx.model_config + num_depths = mc.num_nextn_predict_layers + num_depths = 1 if num_depths is None else int(num_depths) + layer_ids = draft_state_layer_ids(mc, num_depths=num_depths) + if num_depths > 1: + return (ctx.runner,) if ctx.runner in layer_ids else () + return layer_ids + + +def _bind_state(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding: + """This runner's recurrent-state block: its request table is the target's + (same request->slot mappings, same slot allocator) with a `mamba_pool` + view keyed by the runner's own state layer ids.""" + hosts = ctx.hosts_of("state") + assert hosts == ("mamba",), ( + f"the state binder serves the 'mamba' host alone; got {hosts}" + ) + assert isinstance(binding.req_to_token_pool, UnifiedHybridReqToTokenPool), ( + "a fused draft state block needs the target's unified state pool" + ) + layer_ids = _runner_state_layer_ids(ctx) + lanes = ctx.placement.lanes_for(ctx.runner, "mamba") + assert len(layer_ids) == len(lanes), ( + f"draft runner {ctx.runner}: state layers {list(layer_ids)} vs placed " + f"lanes {list(lanes)}" + ) + req_to_token_pool = binding.req_to_token_pool.clone_for_fused_draft( + layer_lanes=dict(zip(layer_ids, lanes)) + ) + return msgspec.structs.replace(binding, req_to_token_pool=req_to_token_pool) + + +register_draft_binder("state", _bind_state) diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index cdebc8b996..e7766b292a 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -25,6 +25,7 @@ compaction only mutates those (no reference rewriting). from __future__ import annotations +import copy import logging from abc import ABC, abstractmethod from dataclasses import dataclass @@ -40,6 +41,8 @@ from sglang.srt.environ import envs from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, + DraftRegion, + DraftStateRegion, FusedDraftPlacement, ) from sglang.srt.mem_cache.layout.token_major import ( @@ -119,7 +122,9 @@ class SubPoolSpec(ABC): layer_num: int grow_direction: str # "up" | "down" | "float" # Fused draft region riding in this sub-pool's entries; None = unfused. - draft_region: Optional[DenseDraftRegion] = None + # Each kind accepts the region kind its entry can lay out, after the host + # block at `draft_offset_in_entry()`. + draft_region: Optional[DraftRegion] = None def __post_init__(self): assert self.grow_direction in self._allowed_grow_directions, ( @@ -176,6 +181,9 @@ class MHASubPoolSpec(SubPoolSpec): f"v_head_dim must be positive; got {self.v_head_dim}" ) if self.draft_region is not None: + assert isinstance(self.draft_region, DenseDraftRegion), ( + "dense entries carry dense draft K/V rows, never a state region" + ) self.draft_region.validate() def k_row_bytes(self) -> int: @@ -258,6 +266,9 @@ class MLASubPoolSpec(SubPoolSpec): f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}" ) if self.draft_region is not None: + assert isinstance(self.draft_region, DenseDraftRegion), ( + "dense entries carry dense draft K/V rows, never a state region" + ) self.draft_region.validate() @property @@ -305,7 +316,15 @@ class MLASubPoolSpec(SubPoolSpec): @dataclass(frozen=True, kw_only=True) class MambaSubPoolSpec(SubPoolSpec): - """Per-slot layout of one Mamba-shaped sub-pool.""" + """Per-slot layout of one Mamba-shaped sub-pool. + + With `draft_region` (a `DraftStateRegion`) set, the draft's state block + follows the host's inside every slot's entry: + + [ conv[0] x L | ... | temporal x L | draft conv[0] x Ld | ... | pad ] + + Unfused, the entry keeps its raw (unaligned) size, byte-identical to before. + """ conv_state_shapes: Tuple[Tuple[int, ...], ...] # one shape per conv tensor conv_dtype: torch.dtype @@ -316,9 +335,11 @@ class MambaSubPoolSpec(SubPoolSpec): def __post_init__(self): super().__post_init__() assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty" - assert self.draft_region is None, ( - "mamba state pages carry no fused draft region yet" - ) + if self.draft_region is not None: + assert isinstance(self.draft_region, DraftStateRegion), ( + "mamba state entries carry a draft STATE region, never dense K/V rows" + ) + self.draft_region.validate() def conv_row_bytes(self, idx: int) -> int: return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize @@ -326,13 +347,26 @@ class MambaSubPoolSpec(SubPoolSpec): def temporal_row_bytes(self) -> int: return _prod(self.temporal_state_shape) * self.temporal_dtype.itemsize - def entry_bytes(self) -> int: + def host_entry_bytes(self) -> int: + """Host (target-only) bytes for one slot, before any draft block.""" total = 0 for i in range(len(self.conv_state_shapes)): total += self.layer_num * self.conv_row_bytes(i) total += self.layer_num * self.temporal_row_bytes() return total + def draft_offset_in_entry(self) -> int: + """Byte offset of the fused draft state block inside one slot's entry.""" + assert self.draft_region is not None + return align_part_offset(self.host_entry_bytes()) + + def entry_bytes(self) -> int: + if self.draft_region is None: + return self.host_entry_bytes() + return align_entry_bytes( + self.draft_offset_in_entry() + self.draft_region.entry_bytes() + ) + def get_dtype(self) -> torch.dtype: return self.conv_dtype # representative state dtype; matches MambaPool.dtype @@ -432,11 +466,7 @@ class UnifiedKVPool: self._bs1_floor_terms = bs1_floor_terms or [] self.fused_draft = fused_draft for spec in sub_pool_specs: - expected = ( - fused_draft.region - if fused_draft is not None and spec.name == "full" - else None - ) + expected = None if fused_draft is None else fused_draft.region(spec.name) assert spec.draft_region is expected, ( f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does " f"not match the fused draft placement's {expected}" @@ -749,6 +779,30 @@ class UnifiedKVPool: temporal_dtype=spec.temporal_dtype, max_slots=max_slots, anchor_bytes=anchor_bytes, + entry_bytes=spec.entry_bytes(), + ) + + def build_draft_state_views( + self, sub_pool_name: str + ) -> Tuple[List[torch.Tensor], torch.Tensor]: + """Per-layer conv/temporal views of the DRAFT state block fused into + ``sub_pool_name``'s entries: same slots, same v2p table as the host.""" + spec = self.mamba_spec(sub_pool_name) + region = spec.draft_region + assert isinstance(region, DraftStateRegion), ( + f"sub-pool {sub_pool_name!r} carries no fused draft state region" + ) + return build_mamba_entry_views( + self._raw, + layer_num=region.lane_num, + conv_state_shapes=region.state.conv_state_shapes, + conv_dtype=region.state.conv_dtype, + temporal_state_shape=region.state.temporal_state_shape, + temporal_dtype=region.state.temporal_dtype, + max_slots=self.max_slots(sub_pool_name), + anchor_bytes=self._anchor_bytes[sub_pool_name], + entry_bytes=spec.entry_bytes(), + offset_bytes=spec.draft_offset_in_entry(), ) @@ -1126,8 +1180,29 @@ class UnifiedMambaPool(MambaPool): self.num_mamba_layers, ) - # Inherited MambaPool state ops (copy_from/clear_slots/get_cpu_copy/load_cpu_copy) - # take PHYSICAL slot ids; callers translate via the slot allocator first. + # State ops take PHYSICAL slot ids and act on whole entries, so a fused + # draft state block is cleared, copied and moved with the host streams. + + def _entry_view(self) -> torch.Tensor: + spec = self._unified_buffer.mamba_spec(self._sub_pool_name) + entry_bytes = spec.entry_bytes() + anchor = self._unified_buffer.anchor_bytes(self._sub_pool_name) + num_slots = self._max_size + 1 + raw = self._unified_buffer._raw + entries = raw[anchor : anchor + num_slots * entry_bytes] + return entries.view(num_slots, entry_bytes) + + def clear_slots(self, indices: torch.Tensor): + self._entry_view()[indices] = 0 + for sibling in self._slot_siblings: + sibling.reset_slots(indices) + + def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor): + assert self.replayssm_write_pos is None + entries = self._entry_view() + entries[dst_indices] = entries[src_indices] + for sibling in self._slot_siblings: + sibling.copy_slots(src_indices, dst_indices) def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor): # Compaction moves disjoint physical slots. Copy their whole envelopes @@ -1143,6 +1218,8 @@ class UnifiedMambaPool(MambaPool): self._unified_buffer.max_slots(self._sub_pool_name), self._unified_buffer.mamba_spec(self._sub_pool_name).entry_bytes(), ) + if self.replayssm_write_pos is not None: + self.replayssm_write_pos[tgt_loc] = 0 for sibling in self._slot_siblings: sibling.copy_slots(src_loc, tgt_loc) @@ -1407,6 +1484,27 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool): ) ) + def clone_for_fused_draft( + self, *, layer_lanes: Dict[int, int] + ) -> UnifiedHybridReqToTokenPool: + """Shallow copy for a draft runner whose recurrent state is fused into + this pool's state entries: it shares the request->slot mappings and + the slot allocator, and its `mamba_pool` is a view of the draft block + keyed by the runner's own layer ids.""" + from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftMambaPool + + clone = copy.copy(self) + clone.mamba_pool = UnifiedDraftMambaPool( + unified_buffer=self._unified_buffer, + sub_pool_name=self._mamba_sub_pool_name, + layer_lanes=layer_lanes, + ) + clone.mamba_map = { + layer_id: idx for idx, layer_id in enumerate(sorted(layer_lanes)) + } + clone.mamba_ckpt_pool = None + return clone + @property def mamba_v2p_table(self) -> Optional[torch.Tensor]: """This pool's ids ARE virtual; page_size is 1, so the translate is the @@ -1552,7 +1650,7 @@ def init_unified_mamba_pools( qk_rope_head_dim=qk_rope_head_dim, store_dtype=store_dtype, grow_direction="down", - draft_region=None if fused_draft is None else fused_draft.region, + draft_region=None if fused_draft is None else fused_draft.region("full"), ) else: full_spec = MHASubPoolSpec( @@ -1564,7 +1662,7 @@ def init_unified_mamba_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="down", - draft_region=None if fused_draft is None else fused_draft.region, + draft_region=None if fused_draft is None else fused_draft.region("full"), ) cp = mamba2_cache_params mamba_spec = MambaSubPoolSpec( @@ -1576,6 +1674,7 @@ def init_unified_mamba_pools( temporal_dtype=cp.dtype.temporal, conv_slice_axis=getattr(cp.shape, "conv_slice_axis", 0), grow_direction="up", + draft_region=None if fused_draft is None else fused_draft.region("mamba"), ) if unified_total_bytes is not None: # PROFILED byte budget for the token side (captured pre-ratio-floor); @@ -2051,10 +2150,10 @@ def init_unified_swa_pools( ) -> UnifiedSWAPoolBundle: """Build the SWA-hybrid unified-memory-pool stack. - With ``fused_draft``, every entry of the "full" sub-pool carries the - draft model's K/V parts after the host parts, and each draft runner - binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a - pool of its own. + With ``fused_draft``, every entry of a sub-pool the placement names + carries the draft model's K/V parts after the host parts, and each draft + runner binds a `UnifiedDraftKVPool` over its own lanes instead of + allocating a pool of its own. """ from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, @@ -2081,7 +2180,7 @@ def init_unified_swa_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="down", - draft_region=None if fused_draft is None else fused_draft.region, + draft_region=None if fused_draft is None else fused_draft.region("full"), ) swa_spec = MHASubPoolSpec( name="swa", @@ -2092,6 +2191,7 @@ def init_unified_swa_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="up", + draft_region=None if fused_draft is None else fused_draft.region("swa"), ) legacy_allocator_capacities = {} if total_bytes is None: @@ -2251,8 +2351,9 @@ def init_unified_mamba_swa_pools( Sizing inputs are the same token counts the 2-pool factories take (ratio- fed until the byte configurator lands); the buffer budget is their byte - sum and the runtime split floats. With ``fused_draft``, every entry of the - "full" sub-pool carries the draft's parts, as in the 2-pool factories. + sum and the runtime split floats. With ``fused_draft``, every entry of a + sub-pool the placement names carries the draft's parts, as in the 2-pool + factories. """ from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedMambaSWATokenToKVPoolAllocator, @@ -2279,7 +2380,7 @@ def init_unified_mamba_swa_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="down", - draft_region=None if fused_draft is None else fused_draft.region, + draft_region=None if fused_draft is None else fused_draft.region("full"), ) swa_spec = MHASubPoolSpec( name="swa", @@ -2290,6 +2391,7 @@ def init_unified_mamba_swa_pools( store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="float", + draft_region=None if fused_draft is None else fused_draft.region("swa"), ) cp = mamba2_cache_params mamba_spec = MambaSubPoolSpec( @@ -2300,6 +2402,7 @@ def init_unified_mamba_swa_pools( temporal_state_shape=tuple(int(x) for x in cp.shape.temporal), temporal_dtype=cp.dtype.temporal, grow_direction="up", + draft_region=None if fused_draft is None else fused_draft.region("mamba"), ) if unified_total_bytes is not None: # PROFILED byte budget for the token side (captured pre-ratio-floor); diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index e5b0e1f6cf..597e536918 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -227,11 +227,12 @@ class UnifiedRadixCache(BasePrefixCache): self.enable_mamba_extra_buffer = ( params.enable_mamba_extra_buffer if self.is_mamba_enabled else False ) - # SWA window size (None when SWA is not enabled). + # Required cache history, shared by retention, budgeting and transfers. self._sliding_window_size = ( - params.sliding_window_size if self.is_swa_enabled else None + params.swa_retain_window if self.is_swa_enabled else None ) self.decoder_swa_bounded_replay = params.decoder_swa_bounded_replay + self.draft_swa_window = params.draft_swa_window # The TreeCore owns the tree member-var state (structure, LRUs, sizes, # evictable leaves) and drives the components' tree-level hooks. self._tree_core_backend = select_tree_core_backend(params) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 95b70b9f63..c4863d45b8 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -739,7 +739,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): self._draft_cell_size = _dflash_draft_cell_size(kvc) self._fused_full_entry = kvc.fused_entry_bytes("full") - if self._fused_full_entry is not None: + self._fused_swa_entry = kvc.fused_entry_bytes("swa") + if self._fused_full_entry is not None or self._fused_swa_entry is not None: self._draft_full_layers_num = 0 self._draft_swa_layers_num = 0 self._draft_swa_full_layers_num = 0 @@ -757,6 +758,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): ) def _swa_cell_bytes(self) -> int: + if self._fused_swa_entry is not None: + return self._fused_swa_entry return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num) def _recompute_cell_size(self) -> None: diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 32fd57626e..5cf87c92ff 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -1841,6 +1841,16 @@ class InklingForConditionalGenerationMTP(nn.Module): hidden_states_before_norm=hidden_states_before_norm, ) + def get_attention_sliding_window_size(self) -> Optional[int]: + """The trunk's convention (`InklingForConditionalGeneration`): the + backend window is one less than the extent, so the head reads back + the same span the trunk's window layers do. A banded depth runs at + its own extent, every other depth at the trunk's window.""" + text_config = self.text_config + if self.draft_model_idx in text_config.mtp_local_layer_ids: + return text_config.mtp_local_extent - 1 + return text_config.sliding_window_size - 1 + def get_embed_and_head(self): return self.model.embed_tokens.weight, self.lm_head.weight diff --git a/python/sglang/srt/speculative/base_spec_worker.py b/python/sglang/srt/speculative/base_spec_worker.py index 891956ca89..c2f4f2a2b4 100644 --- a/python/sglang/srt/speculative/base_spec_worker.py +++ b/python/sglang/srt/speculative/base_spec_worker.py @@ -158,6 +158,12 @@ class BaseSpecWorker(ABC): def hicache_draft_plan(self) -> HiCacheDraftPlan: return self._hicache_draft_plan + @property + def draft_swa_window(self) -> int: + """Tokens back from the committed length that the draft reads in the + target's swa sub-pool; 0 when the target's own window covers them.""" + return 0 + def weight_update_runners(self) -> list[tuple[str, ModelRunner]]: """(role, runner) pairs weight ops apply to; [] when the draft owns no weights.""" # NGRAM and UNO have no draft model diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index 0fa2baa73a..f9a0ae2b5a 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -177,6 +177,10 @@ class EagleDraftInput(SpecInput): # shape: (b + 1,) kv_indptr: torch.Tensor = None kv_indices: torch.Tensor = None + # The window-clipped counterpart of the pair above, for the draft's + # sliding-window layers; None on a pool with no window side. + window_kv_indptr: torch.Tensor = None + window_kv_indices: torch.Tensor = None num_tokens_per_req: int = -1 num_tokens_for_logprob_per_req: int = -1 diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index d1b7050720..ccbf75837b 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -34,6 +34,7 @@ from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool +from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftSWAKVPool from sglang.srt.model_executor.cuda_graph_config import ( Backend, Phase, @@ -587,14 +588,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): return seen = set() for runner in self.draft_runner_list: - pool = runner.req_to_token_pool.mamba_pool + req_pool = runner.req_to_token_pool + pool = req_pool.mamba_pool if id(pool) in seen: continue seen.add(id(pool)) + # The pool is a PHYSICAL store; a unified req pool hands out + # virtual slot ids (identity on a static pool). if clear is not None and len(clear) > 0: - pool.clear_slots(clear) + pool.clear_slots(req_pool.translate_mamba_indices(clear)) if cow_src is not None and len(cow_src) > 0: - pool.copy_from(cow_src, cow_dst) + pool.copy_from( + req_pool.translate_mamba_indices(cow_src), + req_pool.translate_mamba_indices(cow_dst), + ) forward_batch.mamba_clear_indices = None forward_batch.mamba_cow_src_indices = None forward_batch.mamba_cow_dst_indices = None @@ -1092,6 +1099,20 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker): ), ) + @property + def draft_swa_window(self) -> int: + # The widened extend starts front rows below the committed length, and + # each depth reads its own window back from there. + draft = self._draft_worker + windows = [ + runner.sliding_window_size + for runner in draft.draft_runner_list + if isinstance(runner.token_to_kv_pool, UnifiedDraftSWAKVPool) + ] + if not windows: + return 0 + return max(windows) + draft.draft_extend_num_front_tokens + def forward_batch_generation( self, batch: ScheduleBatch, diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index f72644a24d..df5f983b41 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -411,6 +411,11 @@ class SpecInput(ABC): future_dsa_topk_indices_available: bool = False dsa_seed_topk_capture: Optional[torch.Tensor] = None + # A multi-step draft's per-step sliding-window read rail; only the + # multi-step draft backend of a sliding-window KV pool builds it. + window_kv_indptr: Optional[torch.Tensor] = None + window_kv_indices: Optional[torch.Tensor] = None + def __init__(self, spec_input_type: SpecInputType): self.spec_input_type = spec_input_type diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index ab2a3ccb4a..59b7c50cbb 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -23,6 +23,9 @@ from sglang.kernels.ops.speculative.cache_locs import ( from sglang.kernels.ops.speculative.cache_locs import ( generate_draft_decode_kv_indices as generate_draft_decode_kv_indices, ) +from sglang.kernels.ops.speculative.cache_locs import ( + generate_draft_decode_window_kv_indices as generate_draft_decode_window_kv_indices, +) from sglang.kernels.ops.speculative.cache_locs import ( get_src_tgt_cache_loc as get_src_tgt_cache_loc, ) diff --git a/test/registered/kernels/ops/speculative/test_generate_draft_decode_window_kv_indices.py b/test/registered/kernels/ops/speculative/test_generate_draft_decode_window_kv_indices.py new file mode 100644 index 0000000000..0098833ce5 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_generate_draft_decode_window_kv_indices.py @@ -0,0 +1,236 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`generate_draft_decode_window_kv_indices`: the multi-step draft's per-step +sliding-window read rail. + +A draft's window layer at step s attends the last `min(seq_len + s + 1, +window)` tokens of its chain: the committed prefix followed by the draft's own +uncommitted tokens, which live in the chain slots the full kernel emits, not +in req_to_token's live prefix. Pinned: the emitted ids are exactly that +window-clipped tail of ``prefix ++ chain``, translated through the read +source's v2p when it has one; the indptr is the clipped cumsum; TRANSLATE=False +emits the raw ids; and over the plan's physical page table the kernel emits +what the translated gather of the virtual rows does. + + python -m pytest test/registered/kernels/ops/speculative/test_generate_draft_decode_window_kv_indices.py -v +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="nightly", runner_config="1-gpu-large") + +_SENTINEL = -7 + + +def _run_kernel( + *, + table, + seq_lens, + positions, + num_steps, + topk, + page_size, + window, + v2p=None, + entry_page_size=1, +): + from sglang.kernels.ops.speculative.cache_locs import ( + generate_draft_decode_window_kv_indices, + ) + from sglang.srt.utils import next_power_of_2 + + num_seqs = seq_lens.numel() + bs = num_seqs * topk + width = bs * window + out = torch.full((num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda") + indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda") + generate_draft_decode_window_kv_indices[(num_steps, num_seqs, topk)]( + torch.arange(num_seqs, dtype=torch.int64, device="cuda"), + table, + seq_lens, + out, + indptr, + positions, + table.stride(0), + width, + indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(num_steps), + next_power_of_2(bs), + page_size, + window, + ENTRY_PAGE_SIZE=entry_page_size, + v2p=v2p, + TRANSLATE=v2p is not None, + ) + return out, indptr + + +def _chain_slot(seq_len, topk_id, step, *, num_steps, topk, page_size): + """Column of chain token ``step`` of branch ``topk_id`` in a req_to_token + row, as `generate_draft_decode_kv_indices` lays the chain out.""" + if page_size == 1 or topk == 1: + return seq_len + topk_id * num_steps + step + last_page_len = seq_len % page_size + num_new_pages_per_topk = (last_page_len + num_steps + page_size - 1) // page_size + prefix_base = seq_len // page_size * page_size + return ( + prefix_base + + topk_id * num_new_pages_per_topk * page_size + + last_page_len + + step + ) + + +def _reference(*, rows, seq_lens, num_steps, topk, page_size, window, v2p): + """Per step: the window-clipped tail of prefix ++ chain per branch.""" + steps = [] + for s in range(num_steps): + iters = s + 1 + ids, indptr = [], [0] + for b, seq_len in enumerate(seq_lens): + for t in range(topk): + total = seq_len + iters + start = total - min(total, window) + chain = [ + rows[b][ + _chain_slot( + seq_len, + t, + j, + num_steps=num_steps, + topk=topk, + page_size=page_size, + ) + ] + for j in range(iters) + ] + tail = (list(rows[b][:seq_len]) + chain)[start:] + if v2p is not None: + tail = [ + max(v2p[i // page_size] * page_size + i % page_size, 0) + for i in tail + ] + ids.extend(tail) + indptr.append(len(ids)) + steps.append((ids, indptr)) + return steps + + +class TestGenerateDraftDecodeWindowKVIndices(CustomTestCase): + def _check(self, *, page_size, topk, window, translate, num_steps=3): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + torch.manual_seed(3) + num_seqs, max_context = 3, 64 + # Prefix ids and chain slots are all distinct so a misplaced read shows. + req_to_token = torch.randperm(1 << 12, device="cuda")[: num_seqs * max_context] + req_to_token = req_to_token.view(num_seqs, max_context).to(torch.int64) + seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda") + positions = seq_lens.repeat_interleave(topk) + v2p = None + if translate: + num_pages = (1 << 12) // page_size + 1 + perm = torch.randperm(num_pages, device="cuda") + v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda") + v2p[perm] = torch.arange(num_pages, device="cuda") + out, indptr = _run_kernel( + table=req_to_token, + seq_lens=seq_lens, + positions=positions, + num_steps=num_steps, + topk=topk, + page_size=page_size, + window=window, + v2p=v2p, + ) + expected = _reference( + rows=req_to_token.tolist(), + seq_lens=seq_lens.tolist(), + num_steps=num_steps, + topk=topk, + page_size=page_size, + window=window, + v2p=None if v2p is None else v2p.tolist(), + ) + for s, (ids, ptr) in enumerate(expected): + self.assertEqual(indptr[s].tolist(), ptr, f"step {s} indptr") + self.assertEqual(out[s, : len(ids)].tolist(), ids, f"step {s} ids") + # Nothing past the used prefix is touched. + self.assertTrue(bool((out[s, len(ids) :] == _SENTINEL).all()), f"step {s}") + + def test_window_clips_the_prefix_and_keeps_the_chain(self): + # window 4 < seq_len 5 + steps: only the tail of the prefix survives, + # and by step 3 the window is chain-heavy. + self._check(page_size=1, topk=1, window=4, translate=False) + + def test_window_wider_than_the_chain_emits_everything(self): + self._check(page_size=1, topk=1, window=64, translate=False) + + def test_translate_uses_the_swa_side_page_table(self): + self._check(page_size=1, topk=1, window=6, translate=True) + self._check(page_size=2, topk=1, window=6, translate=True) + + def test_paged_chain_slots_with_topk(self): + self._check(page_size=2, topk=2, window=5, translate=True) + + def test_the_plan_page_table_matches_the_translated_gather(self): + """Once the iteration's sliding-window table is built, the rail reads + physical pages from it (entry * ps + pos % ps) instead of translating + the virtual rows; both must emit the same ids.""" + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + for page_size, topk in ((1, 1), (2, 1), (2, 2)): + torch.manual_seed(5) + num_seqs, max_context, num_steps = 3, 64, 3 + num_pages = max_context // page_size + # Rows of whole virtual pages in shuffled order over a shuffled v2p. + v2p = torch.randperm(num_pages, device="cuda") + offsets = torch.arange(page_size, device="cuda") + req_to_token = torch.stack( + [ + ( + torch.randperm(num_pages, device="cuda")[:, None] * page_size + + offsets + ).reshape(-1) + for _ in range(num_seqs) + ] + ) + table = v2p[req_to_token[:, ::page_size] // page_size].to(torch.int32) + seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda") + common = dict( + seq_lens=seq_lens, + positions=seq_lens.repeat_interleave(topk), + num_steps=num_steps, + topk=topk, + page_size=page_size, + window=6, + ) + gathered, gathered_indptr = _run_kernel( + table=req_to_token, v2p=v2p, **common + ) + out, out_indptr = _run_kernel( + table=table, entry_page_size=page_size, **common + ) + torch.testing.assert_close(out_indptr, gathered_indptr, rtol=0, atol=0) + torch.testing.assert_close(out, gathered, rtol=0, atol=0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/dllm/test_gemma4_cuda_graph.py b/test/registered/unit/dllm/test_gemma4_cuda_graph.py index f42282311f..6f59a02f3e 100644 --- a/test/registered/unit/dllm/test_gemma4_cuda_graph.py +++ b/test/registered/unit/dllm/test_gemma4_cuda_graph.py @@ -18,11 +18,12 @@ from sglang.srt.models.gemma4_diffusion import DiffusionGemmaTextEmbedding from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase, published_topology -register_cpu_ci(est_time=3, suite="base-a-test-cpu") +register_cpu_ci(est_time=10, suite="base-a-test-cpu") class _ReadStream: reads_are_translated = False + window_kv_kind = IdSpaceKind.FULL def __init__(self): self.tokens = torch.arange(64).view(4, 16) diff --git a/test/registered/unit/layers/attention/test_cuda_graph_window_rail_completeness.py b/test/registered/unit/layers/attention/test_cuda_graph_window_rail_completeness.py new file mode 100644 index 0000000000..22db3f9ac8 --- /dev/null +++ b/test/registered/unit/layers/attention/test_cuda_graph_window_rail_completeness.py @@ -0,0 +1,176 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The multi-step draft's sliding-window rails on the Triton backend. + +Every cuda-graph forward mode must fill the sliding-window READ rail, and each +draft step must write its window layers through its OWN columns of the +iteration's write window. + +BUG REGRESSION. `_build_cuda_graph_forward_metadata` builds one +`ForwardMetadata` per captured forward mode. The draft-extend branch passed +`window_kv_indices=None` while its buffer-fill helper never built the rail, +so a banded MTP depth (a window layer on the draft) reached the Triton extend +kernel with a null index tensor and died during graph capture as +`AttributeError("'NoneType' object has no attribute 'type'")` inside IR +construction -- a message that names neither the rail nor the mode. The eager +path built it, so only cuda-graph runs failed. + +A bare `None` here is the defect: every mode this function serves can carry +window layers, so each must pass the buffer, guarded on the model actually +having a window. This is an AST check over the real function body, so a new +mode branch that forgets the rail fails here rather than on a GPU. + + python -m pytest test/registered/unit/layers/attention/test_cuda_graph_window_rail_completeness.py -v +""" + +import ast +import unittest +from pathlib import Path +from types import SimpleNamespace + +import torch + +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind, KVLocPlan +from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +_BUILDER = "_build_cuda_graph_forward_metadata" +_RAIL = "window_kv_indices" + + +def _builder_body(): + import sglang.srt.layers.attention.triton_backend as tb + + tree = ast.parse(Path(tb.__file__).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == _BUILDER: + return node + raise AssertionError(f"{_BUILDER} not found in triton_backend.py") + + +def _rail_args(body): + """(line, value node) of the window-rail kwarg of each ForwardMetadata.""" + for call in ast.walk(body): + if not isinstance(call, ast.Call): + continue + name = call.func.id if isinstance(call.func, ast.Name) else None + if name != "ForwardMetadata": + continue + for kw in call.keywords: + if kw.arg == _RAIL: + yield call.lineno, kw.value + break + else: + yield call.lineno, None # kwarg omitted entirely + + +class TestCudaGraphWindowRailCompleteness(CustomTestCase): + def test_no_captured_mode_passes_a_bare_none_rail(self): + rails = list(_rail_args(_builder_body())) + # A scan that finds nothing would pass vacuously forever. + self.assertGreaterEqual( + len(rails), 3, "expected one ForwardMetadata per captured forward mode" + ) + offenders = [ + line + for line, value in rails + if value is None + or (isinstance(value, ast.Constant) and value.value is None) + ] + self.assertEqual( + offenders, + [], + f"{_BUILDER}: {_RAIL} is a bare None at line(s) {offenders}; a window " + "layer in that mode reaches the Triton kernel with a null index " + "tensor. Pass the cuda-graph buffer, guarded on the sliding-window flag.", + ) + + +class _Reader: + """A reader whose sliding-window ids are the virtual ids times ten.""" + + def __init__(self): + self._spaces = { + IdSpaceKind.FULL: IdSpace(key="full"), + IdSpaceKind.SLIDING_WINDOW: IdSpace(key="swa", write=lambda ids: ids * 10), + } + + def space(self, kind): + return self._spaces.get(kind) + + +class TestDraftStepWriteColumns(CustomTestCase): + """BUG REGRESSION. A draft step's per-step backend took its swa write ids + from the whole draft window, and the write then kept their first + ``bs * topk``: step 0's slots for request 0, step 1's for request 0, and so + on, since the window lays each row out step-minor. Each step must bind its + own columns while its backend derives the ids.""" + + def _check(self, *, bs, topk, num_steps, window_cols): + from sglang.srt.layers.attention.triton_backend import ( + TritonMultiStepDraftBackend, + ) + + reader = _Reader() + plan = KVLocPlan( + source=reader, + req_pool_indices=torch.arange(bs), + seq_lens=torch.ones(bs, dtype=torch.int64), + seq_lens_cpu=None, + write_virtual=torch.arange(1, bs * topk * window_cols + 1), + read_extent=0, + ) + # A chain draft (topk 1) inside a wider verify window writes its + # leading columns; otherwise the draft owns the whole window. + cols = None if window_cols == num_steps else slice(0, num_steps) + batch = SimpleNamespace( + batch_size=bs, + kv_loc_plan=plan, + kv_loc_cols=cols, + out_cache_loc=plan.write_ids(reader, cols=cols), + ) + backend = object.__new__(TritonMultiStepDraftBackend) + backend.topk = topk + backend.speculative_num_steps = num_steps + backend.attn_backends = [ + SimpleNamespace(use_sliding_window_kv_pool=True) + ] * num_steps + swa = plan.write_ids(reader, kind=IdSpaceKind.SLIDING_WINDOW, cols=cols) + per_step = per_step_draft_out_cache_loc(swa, bs, topk, num_steps) + for step in range(num_steps): + with backend._step_writes(batch, step): + ids = plan.write_ids( + reader, kind=IdSpaceKind.SLIDING_WINDOW, cols=batch.kv_loc_cols + ) + torch.testing.assert_close(ids, per_step[step], rtol=0, atol=0) + torch.testing.assert_close( + batch.out_cache_loc * 10, ids, rtol=0, atol=0 + ) + self.assertIs(batch.kv_loc_cols, cols) + # The whole window's leading ids are not step 1's slots. + self.assertFalse(torch.equal(swa[: bs * topk], per_step[1])) + + def test_the_draft_window(self): + self._check(bs=3, topk=1, num_steps=3, window_cols=3) + self._check(bs=2, topk=2, num_steps=3, window_cols=3) + + def test_a_chain_draft_inside_the_verify_window(self): + self._check(bs=3, topk=1, num_steps=3, window_cols=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/attention/test_fused_draft_window_metadata.py b/test/registered/unit/layers/attention/test_fused_draft_window_metadata.py new file mode 100644 index 0000000000..e154e9680f --- /dev/null +++ b/test/registered/unit/layers/attention/test_fused_draft_window_metadata.py @@ -0,0 +1,223 @@ +"""Window drafts keep their attention mask when placement falls back to FULL. + +The old multi-step path only built window indices for SWA pools, so accepted +FULL placements reached decode with no window metadata. Exercise placement, +binding and the real index reader through eager and captured metadata gathers. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.attention import triton_backend as tb +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWATokenToKVPoolAllocator, +) +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.layout.fused_draft import ( + LAYER_WINDOW, + DraftKVGeometry, + DraftKVProfile, + DraftLayerSet, + PlacementContext, + place_fused_draft, +) +from sglang.srt.mem_cache.unified_draft_pool import bind_fused_draft +from sglang.srt.mem_cache.unified_memory_pool import ( + MHASubPoolSpec, + UnifiedKVPool, + UnifiedSWAKVPool, +) +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-small") + + +class TestFusedDraftWindowMetadata(CustomTestCase): + def setUp(self): + reset_context() + self.addCleanup(reset_context) + self.args = ServerArgs(model_path="dummy", page_size=1) + publish(self.args, role="tokenizer") + + def _fixture(self, window, target_window): + decision = place_fused_draft( + profile=DraftKVProfile( + num_layers=1, + num_depths=1, + kinds={ + LAYER_WINDOW: DraftLayerSet( + layer_ids=(0,), + geometry=DraftKVGeometry(head_num=1, head_dim=8, v_head_dim=8), + window=window, + ) + }, + ), + num_runners=1, + ctx=PlacementContext( + host_names=("full", "swa") if target_window else ("full", "mamba"), + target_window=target_window, + asymmetric_rows_ok=False, + draft_backends=("triton",), + ), + store_dtype=torch.bfloat16, + ) + self.assertIsNotNone(decision.placement, decision.declined) + placement = decision.placement + host = "swa" if target_window and window <= target_window else "full" + self.assertEqual(placement.hosts(), (host,)) + specs = [ + MHASubPoolSpec( + name=name, + layer_num=1, + head_num=1, + head_dim=8, + store_dtype=torch.bfloat16, + grow_direction=direction, + draft_region=placement.region(name), + ) + for name, direction in (("full", "up"), ("swa", "down")) + ] + buffer = UnifiedKVPool( + total_bytes=1024 * sum(spec.entry_bytes() for spec in specs), + sub_pool_specs=specs, + device="cuda", + enable_memory_saver=False, + page_size=1, + fused_draft=placement, + ) + target_pool = UnifiedSWAKVPool( + unified_buffer=buffer, + full_attention_layer_ids=[0], + swa_attention_layer_ids=[1], + ) + allocator = UnifiedSWATokenToKVPoolAllocator( + unified_buffer=buffer, + kvcache=target_pool, + device="cuda", + full_max_total_num_tokens=1024, + swa_max_total_num_tokens=1024, + ) + # Different FULL/SWA addresses make a wrong storage route observable. + old = allocator.alloc(7) + allocator.free_swa_segment(old, start_pos=0) + virtual = allocator.alloc(512) + req = SimpleNamespace(size=1, req_to_token=virtual.reshape(1, -1)) + layer = RadixAttention( + num_heads=1, + head_dim=8, + scaling=0.5, + num_kv_heads=1, + layer_id=0, + sliding_window_size=window, + ) + bound = bind_fused_draft( + placement=placement, + runner=0, + unified_buffer=buffer, + host_allocator=allocator, + model=SimpleNamespace(modules=lambda: [layer]), + model_config=SimpleNamespace(), + req_to_token_pool=req, + page_size=1, + ) + reader = KVIndexTranslator( + req_to_token=req.req_to_token, + token_to_kv_pool_allocator=allocator, + token_to_kv_pool=bound.token_to_kv_pool, + page_size=1, + device="cuda", + is_draft_worker=True, + ) + runner = SimpleNamespace( + req_to_token_pool=req, + device="cuda", + model_config=SimpleNamespace(get_max_num_attention_heads=lambda: 1), + server_args=self.args, + sliding_window_size=window, + kv_index_translator=reader, + token_to_kv_pool=bound.token_to_kv_pool, + ) + # The model's per-step attention is outside this metadata test. All + # placement, allocation, translation and multi-step metadata are real. + inner = SimpleNamespace(max_context_len=512) + with patch.object(tb, "TritonAttnBackend", return_value=inner): + backend = tb.TritonMultiStepDraftBackend(runner, 1, 3) + rows = torch.zeros(1, dtype=torch.int64, device="cuda") + seq_lens = torch.tensor([300], dtype=torch.int32, device="cuda") + plan = reader.plan( + req_pool_indices=rows, + seq_lens=seq_lens, + seq_lens_cpu=None, + write_virtual=None, + read_extent=3, + ) + batch = SimpleNamespace( + batch_size=1, + seq_lens_sum=None, + seq_lens=seq_lens, + req_pool_indices=rows, + positions=seq_lens.to(torch.int64), + kv_loc_plan=plan, + spec_info=SimpleNamespace(window_kv_indices=None, window_kv_indptr=None), + ) + full = allocator.translate_kv_loc(virtual) + swa = allocator.translate_loc_from_full_to_swa(virtual) + self.assertFalse(torch.equal(full, swa)) + return backend, batch, full if host == "full" else swa + + def test_window_reads_follow_placement_in_eager_and_graph(self): + for window, target_window in ((256, 128), (64, None), (64, 128)): + with self.subTest(window=window, target_window=target_window): + backend, batch, physical = self._fixture(window, target_window) + full_ids = torch.empty((3, 512), dtype=torch.int64, device="cuda") + window_ids = torch.zeros((3, window), dtype=torch.int64, device="cuda") + observed = [] + + def collect(step, fb): + self.assertIsNotNone(fb.spec_info.window_kv_indices) + observed.append( + ( + step, + fb.spec_info.window_kv_indptr.clone(), + fb.spec_info.window_kv_indices.clone(), + ) + ) + + def gather(): + backend.common_template(batch, full_ids, collect, window_ids) + + def check(length): + for step, indptr, indices in observed: + end = length + step + 1 + torch.testing.assert_close( + indptr, + torch.tensor([0, window], dtype=torch.int32, device="cuda"), + ) + torch.testing.assert_close( + indices, physical[end - window : end] + ) + + gather() + check(300) + observed.clear() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + gather() + torch.cuda.current_stream().wait_stream(stream) + batch.seq_lens.fill_(301) + batch.positions.fill_(301) + graph.replay() + check(301) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py index 3b67db423a..1aa58a9454 100644 --- a/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py +++ b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py @@ -13,11 +13,15 @@ slot, and a host page move carries the draft bytes with it. import unittest from types import SimpleNamespace +from unittest.mock import patch import torch +from sglang.srt.mem_cache import memory_pool as mem_pool from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, + DraftStateGeometry, + DraftStateRegion, FusedDraftPlacement, ) from sglang.srt.mem_cache.layout.token_major import ( @@ -25,19 +29,25 @@ from sglang.srt.mem_cache.layout.token_major import ( align_entry_bytes, align_part_offset, ) -from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.unified_draft_pool import ( + UnifiedDraftKVPool, + UnifiedDraftMambaPool, + UnifiedDraftSWAKVPool, +) from sglang.srt.mem_cache.unified_memory_pool import ( MambaSubPoolSpec, MHASubPoolSpec, MLASubPoolSpec, UnifiedKVPool, + UnifiedMambaPool, UnifiedMHATokenToKVPool, ) from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=7, suite="base-a-test-cpu") +register_cpu_ci(est_time=10, suite="base-a-test-cpu") _DEV = "cpu" _DTYPE = torch.bfloat16 @@ -63,8 +73,35 @@ def _draft_region(): ) -def _placement(region): - return FusedDraftPlacement(region=region, runner_lane_counts=(region.lane_num,)) +def _state_region(lane_num=1): + return DraftStateRegion( + lane_num=lane_num, + state=DraftStateGeometry( + conv_state_shapes=((2, 4), (2, 4)), + conv_dtype=torch.float32, + temporal_state_shape=(2,), + temporal_dtype=torch.float32, + ), + ) + + +def _mamba_spec(draft_region=None, layer_num=2): + return MambaSubPoolSpec( + name="mamba", + layer_num=layer_num, + conv_state_shapes=((3, 8),), + conv_dtype=torch.bfloat16, + temporal_state_shape=(2,), + temporal_dtype=torch.float32, + grow_direction="up", + draft_region=draft_region, + ) + + +def _placement(region, num_runners=1): + return FusedDraftPlacement.from_counts( + counts={"full": [region.lane_num] * num_runners}, regions={"full": region} + ) class TestFusedSpecMath(unittest.TestCase): @@ -270,6 +307,144 @@ class TestUnifiedDraftKVPool(unittest.TestCase): torch.testing.assert_close(dp.get_key_buffer(0)[loc].float(), k) +class TestFusedStateHost(unittest.TestCase): + """A state (mamba) entry carries the draft's state block after the host's + streams: the same aligned entry for both, byte-disjoint blocks within a + slot, and the host's whole-entry copy/clear carry the draft block, so a + radix checkpoint restores the draft's state with the target's. Unfused, + the entry stays its raw size; that identity guards every state deploy.""" + + SLOTS = 8 + + def test_unfused_spec_is_byte_identical_to_before(self): + s = _mamba_spec() + self.assertEqual(s.host_entry_bytes(), 2 * (3 * 8 * 2 + 2 * 4)) + self.assertEqual(s.entry_bytes(), s.host_entry_bytes()) + + def test_fused_entry_appends_the_state_block(self): + f = _mamba_spec(_state_region()) + host = f.host_entry_bytes() + self.assertEqual(f.draft_offset_in_entry(), align_part_offset(host)) + self.assertEqual( + f.entry_bytes(), + align_entry_bytes(f.draft_offset_in_entry() + f.draft_region.entry_bytes()), + ) + self.assertEqual(f.draft_region.entry_bytes(), 2 * (2 * 4 * 4) + 2 * 4) + + def _pool(self): + region = _state_region() + mamba = _mamba_spec(region) + full = _host_spec() + total = 4 * full.entry_bytes() + self.SLOTS * mamba.entry_bytes() + pool = UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[full, mamba], + device=_DEV, + enable_memory_saver=False, + page_size=1, + fused_draft=FusedDraftPlacement.from_counts( + counts={"mamba": [1]}, regions={"mamba": region} + ), + ) + return mamba, pool + + def test_host_and_draft_blocks_stay_byte_disjoint_within_a_slot(self): + spec, pool = self._pool() + host_conv, host_temporal = pool.mamba_views_for("mamba") + draft_conv, draft_temporal = pool.build_draft_state_views("mamba") + raw = pool._raw + entry = spec.entry_bytes() + slot = 3 + lo = slot * entry + split = lo + spec.draft_offset_in_entry() + hi = (slot + 1) * entry + for view in (*host_conv, host_temporal): + for layer in range(view.shape[0]): + raw.zero_() + view[layer][slot] = 1.0 + nz = raw.nonzero() + self.assertGreater(nz.numel(), 0) + self.assertTrue(bool((nz >= lo).all() and (nz < split).all())) + for view in (*draft_conv, draft_temporal): + for layer in range(view.shape[0]): + raw.zero_() + view[layer][slot] = 1.0 + nz = raw.nonzero() + self.assertGreater(nz.numel(), 0) + self.assertTrue(bool((nz >= split).all() and (nz < hi).all())) + + def test_host_whole_entry_ops_carry_the_draft_block(self): + _, pool = self._pool() + host = UnifiedMambaPool( + unified_buffer=pool, + sub_pool_name="mamba", + spec_state_size=2, + mamba_layer_ids=[0, 1], + ) + draft_conv, _ = pool.build_draft_state_views("mamba") + host.mamba_cache.conv[0][1][3] = 2.0 + draft_conv[1][0][3] = 7.0 + host.copy_from(torch.tensor([3]), torch.tensor([5])) + self.assertEqual(float(host.mamba_cache.conv[0][1][5].sum()), 2.0 * 3 * 8) + self.assertEqual(float(draft_conv[1][0][5].sum()), 7.0 * 2 * 4) + host.clear_slots(torch.tensor([5])) + self.assertEqual(float(draft_conv[1][0][5].sum()), 0.0) + self.assertEqual(float(host.mamba_cache.conv[0][1][5].sum()), 0.0) + host.move_kv_cache(torch.tensor([6]), torch.tensor([3])) + self.assertEqual(float(host.mamba_cache.conv[0][1][6].sum()), 2.0 * 3 * 8) + self.assertEqual(float(draft_conv[1][0][6].sum()), 7.0 * 2 * 4) + + def test_draft_views_alias_the_fused_block(self): + _, pool = self._pool() + draft = UnifiedDraftMambaPool( + unified_buffer=pool, sub_pool_name="mamba", layer_lanes={4: 0} + ) + draft_conv, _ = pool.build_draft_state_views("mamba") + self.assertEqual( + draft.mamba_cache.conv[1].shape, (1, pool.max_slots("mamba"), 2, 4) + ) + self.assertEqual( + draft.mamba2_layer_cache(0).conv[1].data_ptr(), + draft_conv[1][0].data_ptr(), + ) + + def test_slot_ops_clear_the_draft_block_and_spare_the_host(self): + """BUG REGRESSION. These ops were no-ops, on the theory that the host's + whole-entry clear covers the draft block. It does, but ModelRunner gates + that to a plain EXTEND forward and returns on decode, target-verify and + draft-extend -- exactly the modes a draft needs. A slot handed to a new + request therefore kept its previous occupant's state, and a slot never + written kept whatever the buffer held; a poisoned pool turned that into + NaN and the accept length decayed over a long run (an Inkling MTP head). + Equally, they must NOT clear the whole entry: the host's own streams + share it and stay live while the draft's block is recycled.""" + spec, pool = self._pool() + draft = UnifiedDraftMambaPool( + unified_buffer=pool, sub_pool_name="mamba", layer_lanes={4: 0} + ) + host = UnifiedMambaPool( + unified_buffer=pool, + sub_pool_name="mamba", + spec_state_size=2, + mamba_layer_ids=[0, 1], + ) + draft_conv, draft_temporal = pool.build_draft_state_views("mamba") + host.mamba_cache.conv[0][1][2] = 5.0 # host state, same slot + draft_conv[0][0][2] = 9.0 + draft_temporal[0][2] = 7.0 + + draft.clear_slots(torch.tensor([2])) + self.assertEqual(float(draft_conv[0][0][2].sum()), 0.0) + self.assertEqual(float(draft_temporal[0][2].sum()), 0.0) + # The host's stream in the SAME slot must survive. + self.assertEqual(float(host.mamba_cache.conv[0][1][2].sum()), 5.0 * 3 * 8) + + draft_conv[0][0][2] = 4.0 + draft.copy_from(torch.tensor([2]), torch.tensor([6])) + self.assertEqual(float(draft_conv[0][0][6].sum()), 4.0 * 2 * 4) + self.assertEqual(float(host.mamba_cache.conv[0][1][6].sum()), 0.0) + + def _mla_host_spec(draft_region=None): # 32 B latent rows, two layers: a 64 B host entry before the draft parts. return MLASubPoolSpec( @@ -283,7 +458,14 @@ def _mla_host_spec(draft_region=None): ) -def _mamba_spec(): +def _mamba_filler_spec(): + """A minimal second sub-pool so the MLA host is not alone in the pool. + + Deliberately not `_mamba_spec`: this one is a bystander with its own tiny + geometry, while `_mamba_spec` is the subject of the fused-state tests. Two + module-level defs of one name silently leave only the last, which is how + every `_mamba_spec(region)` call started raising TypeError. + """ return MambaSubPoolSpec( name="mamba", layer_num=1, @@ -325,7 +507,7 @@ class TestFusedMLAHost(unittest.TestCase): def _pool(self): full = _mla_host_spec(_draft_region()) - mamba = _mamba_spec() + mamba = _mamba_filler_spec() total = self.PAGES * self.PS * full.entry_bytes() + 4 * mamba.entry_bytes() return UnifiedKVPool( total_bytes=total, @@ -422,5 +604,101 @@ class TestFusedMLAHost(unittest.TestCase): ) +class TestUnifiedDraftSWAKVPool(unittest.TestCase): + """A draft with window layers binds one dense side per host sub-pool and + routes per layer like the target's composite: a window layer's write + needs the swa loc and lands in the swa entry's draft part, never in the + full entry.""" + + PS = 2 + PAGES = 8 + + def _pool(self): + full_region = _draft_region() + swa_region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + full = _host_spec(full_region) + swa = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=_DTYPE, + grow_direction="up", + draft_region=swa_region, + ) + total = self.PAGES * self.PS * (full.entry_bytes() + swa.entry_bytes()) + return UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[full, swa], + device=_DEV, + enable_memory_saver=False, + page_size=self.PS, + fused_draft=FusedDraftPlacement.from_counts( + counts={"full": [1], "swa": [1]}, + regions={"full": full_region, "swa": swa_region}, + ), + ) + + def _draft_pool(self, pool): + return UnifiedDraftSWAKVPool( + unified_buffer=pool, + host_allocator=object(), + page_size=self.PS, + full_layer_lanes={0: 0}, + swa_layer_lanes={1: 0}, + ) + + def test_routes_each_layer_to_its_side(self): + pool = self._pool() + dp = self._draft_pool(pool) + self.assertEqual(dp.layers_mapping, {0: (0, False), 1: (0, True)}) + dk, _ = pool.build_dense_draft_views("swa") + self.assertEqual(dp.get_key_buffer(1).data_ptr(), dk[0].data_ptr()) + self.assertEqual(dp.get_key_buffer(1).shape[1:], (1, 8)) + self.assertEqual(dp.get_key_buffer(0).shape[1:], (1, 24)) + self.assertEqual(dp.swa_layer_nums, 1) + self.assertEqual(dp.full_layer_nums, 1) + + def test_a_window_only_draft_answers_its_own_v_width(self): + # MiMoV2MTP: no full layer at all; the composite still works. + pool = self._pool() + swa_only = UnifiedDraftSWAKVPool( + unified_buffer=pool, + host_allocator=object(), + page_size=self.PS, + full_layer_lanes={}, + swa_layer_lanes={0: 0}, + ) + self.assertIsNone(swa_only.full_kv_pool) + self.assertEqual(swa_only.get_v_head_dim(), 8) + self.assertEqual(swa_only.layers_mapping, {0: (0, True)}) + + def test_window_write_needs_the_swa_loc_and_stays_in_the_swa_entry(self): + pool = self._pool() + dp = self._draft_pool(pool) + layer = SimpleNamespace(layer_id=1) + k = torch.full((1, 1, 8), 3.0, dtype=_DTYPE) + v = torch.full((1, 1, 8), 5.0, dtype=_DTYPE) + loc = torch.tensor([6], dtype=torch.int64) + with self.assertRaises(AssertionError): + dp.set_kv_buffer(layer, KVWriteLoc(loc, physical=True), k, v) + raw = pool._raw + raw.zero_() + # `_is_cuda` is a PLATFORM constant, not a per-tensor device check, so + # on a CUDA box the store dispatches the CUDA-only `sglang::store_cache` + # at these CPU tensors and raises NotImplementedError. This fixture is + # CPU by contract (register_cpu_ci), and the subject here is WHERE the + # bytes land, not which kernel puts them there -- take the naive path. + with patch.object(mem_pool, "can_use_store_cache", return_value=False): + dp.set_kv_buffer(layer, KVWriteLoc(loc, swa_loc=loc, physical=True), k, v) + swa_entry = pool.spec("swa").entry_bytes() + lo = 6 * swa_entry + pool.spec("swa").draft_offset_in_entry() + nz = raw.nonzero() + self.assertGreater(nz.numel(), 0) + self.assertTrue(bool((nz >= lo).all() and (nz < 7 * swa_entry).all())) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index 3075fcba60..fea2a14eda 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -21,15 +21,20 @@ Pinned: (one shared region would let the runners clobber each other's KV); - a per-depth head serves one depth per runner and needs one runner per depth; - - a draft with SWA or recurrent-state layers of its own declines, and so do - asymmetric K/V rows unless every resolved backend, the draft's included, - carries v_head_dim through to the kernel; + - a draft with a layer kind no registered host kind serves declines, and + so do asymmetric K/V rows unless every resolved backend, the draft's + included, carries v_head_dim through to the kernel; - so does a draft whose attention backend is off the translated MHA rails (it would read the fused rows with virtual ids), a draft under --dcp-size > 1 (each rank's rows hold only its share of the tokens), an explicit draft KV dtype unlike the host's, and a draft under HiCache or host-pool decode retraction (they build the draft's host pool off a - device pool of its own). + device pool of its own); + - the profile divides the draft's heads by attn_tp, as the target does, + and owns recurrent state only for a per-depth conv-chain head; + - the registry refuses a duplicate host or a second primary host for one + layer kind and reports hosts in registration order, every placed host + reports itself in the boot log, and runner ranges must tile a region. python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v """ @@ -43,10 +48,25 @@ import torch from sglang.srt.arg_groups.kv_cache_hook import ASYMMETRIC_KV_BACKENDS from sglang.srt.mem_cache import kv_cache_configurator as kcc from sglang.srt.mem_cache.layout.fused_draft import ( + FULL_HOST, + HOST_KINDS, + LAYER_FULL, + LAYER_STATE, + LAYER_WINDOW, DenseDraftRegion, + DenseHostKind, DraftKVGeometry, DraftKVProfile, + DraftLayerSet, + DraftStateGeometry, + DraftStateRegion, + FusedDraftDecision, + FusedDraftPlacement, + PlacementContext, + RunnerLanes, + draft_kv_profile, place_fused_draft, + register_host_kind, ) from sglang.srt.runtime_context import get_parallel, override_platform from sglang.test.ci.ci_register import register_cpu_ci @@ -54,75 +74,237 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=5, suite="base-a-test-cpu") +_HOSTS = ("full", "swa") +_HOSTS_TRI = ("full", "swa", "mamba") _DTYPE = torch.bfloat16 +_WINDOW = DraftKVGeometry(head_num=2, head_dim=64, v_head_dim=64) +_STATE = DraftStateGeometry( + conv_state_shapes=((2, 4), (2, 4)), + conv_dtype=torch.bfloat16, + temporal_state_shape=(1, 8, 8), + temporal_dtype=torch.float32, +) def _profile( *, num_layers=1, - swa_layer_ids=(), + window_layer_ids=(), num_depths=1, - num_state_layers=0, + state_layer_ids=(), head_dim=64, v_head_dim=64, + window=64, ): - return DraftKVProfile( - num_layers=num_layers, - full=DraftKVGeometry(head_num=4, head_dim=head_dim, v_head_dim=v_head_dim), - swa_layer_ids=swa_layer_ids, - num_depths=num_depths, - num_state_layers=num_state_layers, - ) + kinds = {} + full_ids = tuple(i for i in range(num_layers) if i not in window_layer_ids) + if full_ids: + kinds[LAYER_FULL] = DraftLayerSet( + layer_ids=full_ids, + geometry=DraftKVGeometry( + head_num=4, head_dim=head_dim, v_head_dim=v_head_dim + ), + ) + if window_layer_ids: + kinds[LAYER_WINDOW] = DraftLayerSet( + layer_ids=tuple(window_layer_ids), geometry=_WINDOW, window=window + ) + if state_layer_ids: + kinds[LAYER_STATE] = DraftLayerSet( + layer_ids=tuple(state_layer_ids), geometry=_STATE + ) + return DraftKVProfile(num_layers=num_layers, num_depths=num_depths, kinds=kinds) -def _place(profile, num_runners=1, asymmetric_rows_ok=False): +def _place( + profile, + num_runners=1, + asymmetric_rows_ok=False, + host_names=_HOSTS, + target_window=None, + draft_backends=("triton",), +): return place_fused_draft( profile=profile, num_runners=num_runners, + ctx=PlacementContext( + host_names=host_names, + target_window=target_window, + asymmetric_rows_ok=asymmetric_rows_ok, + draft_backends=draft_backends, + ), store_dtype=_DTYPE, - asymmetric_rows_ok=asymmetric_rows_ok, ) +def _dense_only_registry(): + return patch.dict(HOST_KINDS, {"full": FULL_HOST}, clear=True) + + class TestPlaceFusedDraft(CustomTestCase): def test_replicated_head_gets_one_slot_range_per_runner(self): decision = _place(_profile(num_layers=2), num_runners=3) placement = decision.placement self.assertIsNotNone(placement, decision.declined) - self.assertEqual(placement.region.lane_num, 6) + self.assertEqual(placement.region("full").lane_num, 6) + self.assertEqual(placement.hosts(), ("full",)) for r in range(3): - self.assertEqual(placement.lanes_for(r), range(2 * r, 2 * r + 2)) + self.assertEqual(placement.lanes_for(r, "full"), range(2 * r, 2 * r + 2)) + self.assertEqual(placement.lanes_for(r, "swa"), range(0)) def test_per_depth_head_serves_one_depth_per_runner(self): decision = _place(_profile(num_layers=8, num_depths=8), num_runners=2) placement = decision.placement self.assertIsNotNone(placement, decision.declined) - self.assertEqual(placement.region.lane_num, 2) - self.assertEqual(placement.lanes_for(1), range(1, 2)) + self.assertEqual(placement.region("full").lane_num, 2) + self.assertEqual(placement.lanes_for(1, "full"), range(1, 2)) + + def test_a_tri_pool_host_fuses_a_full_only_draft(self): + decision = _place(_profile(), host_names=("full", "swa", "mamba")) + self.assertIsNotNone(decision.placement, decision.declined) + self.assertEqual(decision.placement.hosts(), ("full",)) + + def test_window_layers_ride_in_the_swa_sub_pool_within_the_target_window(self): + # MiMoV2MTP shape: one window layer per runner, the target's own window. + decision = _place( + _profile(window_layer_ids=(0,), window=128), + num_runners=3, + target_window=128, + ) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertIsNone(decision.note) + self.assertEqual(placement.hosts(), ("swa",)) + self.assertIsNone(placement.region("full")) + self.assertEqual(placement.region("swa").lane_num, 3) + self.assertEqual(placement.region("swa").head_num, _WINDOW.head_num) + for r in range(3): + self.assertEqual(placement.lanes_for(r, "swa"), range(r, r + 1)) + self.assertEqual(placement.lanes_for(r, "full"), range(0)) + + def test_per_depth_head_places_each_depth_by_its_kind(self): + # Inkling shape: depth 1 is a local (window) block, depths 0 and 2 full. + decision = _place( + _profile(num_layers=8, num_depths=8, window_layer_ids=(1,), window=64), + num_runners=3, + target_window=128, + ) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertEqual(placement.region("full").lane_num, 2) + self.assertEqual(placement.region("swa").lane_num, 1) + self.assertEqual(placement.lanes_for(0, "full"), range(0, 1)) + self.assertEqual(placement.lanes_for(1, "swa"), range(0, 1)) + self.assertEqual(placement.lanes_for(1, "full"), range(0)) + self.assertEqual(placement.lanes_for(2, "full"), range(1, 2)) + + def test_window_layers_fall_back_to_the_full_sub_pool(self): + """A window wider than the target's, an undeclared window, or a host + without an swa sub-pool sends the window layers to the full sub-pool + with their own row geometry; the decision says why.""" + for kwargs in ( + dict(window=256, target_window=128), + dict(window=None, target_window=128), + dict(window=64, target_window=None), + dict(window=64, target_window=128, host_names=("full", "mamba")), + ): + profile = _profile(window_layer_ids=(0,), window=kwargs.pop("window")) + decision = _place(profile, num_runners=2, **kwargs) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertIn("ride in the 'full' sub-pool", decision.note) + self.assertIsNone(placement.region("swa")) + self.assertEqual(placement.region("full").lane_num, 2) + self.assertEqual(placement.region("full").head_num, _WINDOW.head_num) + self.assertEqual(placement.lanes_for(1, "full"), range(1, 2)) + + def test_a_fold_cannot_mix_two_row_geometries(self): + """One region holds one row geometry: window layers the swa sub-pool + turns away fold into the full sub-pool only when their rows match the + full layers', else the whole draft declines.""" + decision = _place( + _profile(num_layers=2, window_layer_ids=(1,), window=256), + target_window=128, + ) + self.assertIsNone(decision.placement) + self.assertIn("cannot share", decision.declined) + + def test_a_window_draft_needs_a_rail_carrying_draft_backend(self): + """Only the Triton multi-step draft backend builds the per-step window + rails; any other draft backend would sink-write the window layers.""" + decision = _place( + _profile(window_layer_ids=(0,), window=128), + target_window=128, + draft_backends=("fa3",), + ) + self.assertIsNone(decision.placement) + self.assertIn("['fa3']", decision.declined) + self.assertIn("triton", decision.declined) + # A full-only draft never consults the rail rule. + self.assertIsNotNone(_place(_profile(), draft_backends=("fa3",)).placement) def test_per_depth_head_needs_one_runner_per_depth(self): self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 1).placement) self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 9).placement) - def test_swa_state_asymmetric_and_misaligned_drafts_decline(self): - for profile in ( - _profile(swa_layer_ids=(0,)), - _profile(num_state_layers=1), - _profile(head_dim=64, v_head_dim=32), + def test_state_layers_ride_in_the_mamba_sub_pool(self): + # Inkling shape: one block per depth, every depth carrying conv state. + decision = _place( + _profile(num_layers=8, num_depths=8, state_layer_ids=tuple(range(8))), + num_runners=3, + host_names=_HOSTS_TRI, + ) + placement = decision.placement + self.assertIsNotNone(placement, decision.declined) + self.assertEqual(placement.hosts(), ("full", "mamba")) + self.assertEqual(placement.region("mamba").lane_num, 3) + self.assertEqual(placement.region("mamba").state, _STATE) + self.assertEqual( + placement.region("mamba").entry_bytes(), 3 * _STATE.layer_bytes() + ) + for r in range(3): + self.assertEqual(placement.lanes_for(r, "mamba"), range(r, r + 1)) + # A replicated head carries every state layer per runner. + placement = _place( + _profile(num_layers=2, state_layer_ids=(0, 1)), + num_runners=2, + host_names=_HOSTS_TRI, + ).placement + self.assertEqual(placement.region("mamba").lane_num, 4) + self.assertEqual(placement.lanes_for(1, "mamba"), range(2, 4)) + + def test_state_layers_decline_without_a_state_host(self): + decision = _place(_profile(state_layer_ids=(0,)), host_names=_HOSTS) + self.assertIsNone(decision.placement) + self.assertIn("'mamba'", decision.declined) + + def test_a_layer_kind_with_no_host_kind_declines(self): + with _dense_only_registry(): + for profile, noun in ( + (_profile(num_layers=2, window_layer_ids=(0,)), "sliding-window"), + (_profile(state_layer_ids=(0,)), "recurrent-state"), + ): + decision = _place(profile) + self.assertIsNone(decision.placement) + self.assertIn(f"1 {noun} layer(s)", decision.declined) + + def test_asymmetric_and_misaligned_rows_decline(self): + for profile, reason in ( + (_profile(head_dim=64, v_head_dim=32), "asymmetric"), # 4 heads x 5 dims x 2 B = 40 B: not a 16-B-aligned entry part. - _profile(head_dim=5, v_head_dim=5), + (_profile(head_dim=5, v_head_dim=5), "40 B"), ): decision = _place(profile) self.assertIsNone(decision.placement) - self.assertIsNotNone(decision.declined) + self.assertIn(reason, decision.declined) def test_asymmetric_rows_fuse_with_their_v_width_when_backends_allow(self): placement = _place( _profile(head_dim=64, v_head_dim=32), asymmetric_rows_ok=True ).placement self.assertIsNotNone(placement) - self.assertEqual(placement.region.resolved_v_head_dim(), 32) - self.assertEqual(placement.region.entry_bytes(), 4 * (64 + 32) * 2) + self.assertEqual(placement.region("full").resolved_v_head_dim(), 32) + self.assertEqual(placement.region("full").entry_bytes(), 4 * (64 + 32) * 2) # Symmetric rows never consult the rule. self.assertIsNotNone(_place(_profile(), asymmetric_rows_ok=False).placement) @@ -136,13 +318,90 @@ class TestPlaceFusedDraft(CustomTestCase): def test_region_carries_the_profile_geometry(self): placement = _place(_profile()).placement self.assertEqual( - placement.region, + placement.region("full"), DenseDraftRegion( lane_num=1, head_num=4, head_dim=64, v_head_dim=64, store_dtype=_DTYPE ), ) +def _config(**overrides): + fields = dict( + is_hybrid_swa=True, + is_deepseek_v4_arch=False, + swa_attention_layer_ids=[0], + num_nextn_predict_layers=None, + hf_text_config=SimpleNamespace(), + get_num_kv_heads=lambda tp: max(1, 8 // tp), + head_dim=64, + v_head_dim=32, + get_swa_num_kv_heads=lambda tp: max(1, 4 // tp), + swa_head_dim=64, + swa_v_head_dim=64, + sliding_window_size=128, + ) + fields.update(overrides) + return SimpleNamespace(**fields) + + +_TRUNK_STATE = SimpleNamespace( + mamba2_cache_params=SimpleNamespace( + shape=SimpleNamespace(conv=((2, 4), (2, 4)), temporal=(1, 8, 8)), + dtype=SimpleNamespace(conv=torch.bfloat16, temporal=torch.float32), + layers=list(range(36)), + ) +) + + +class TestDraftKVProfile(CustomTestCase): + def test_heads_are_divided_by_attn_tp(self): + with patch("sglang.srt.configs.hybrid_arch.mambaish_config", return_value=None): + profile = draft_kv_profile(_config(), num_layers=2, attn_tp_size=2) + self.assertEqual(profile.num_depths, 1) + self.assertEqual(set(profile.kinds), {LAYER_FULL, LAYER_WINDOW}) + full = profile.kinds[LAYER_FULL] + self.assertEqual(full.layer_ids, (1,)) + self.assertEqual( + full.geometry, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32) + ) + window = profile.kinds[LAYER_WINDOW] + self.assertEqual(window.layer_ids, (0,)) + self.assertEqual( + window.geometry, DraftKVGeometry(head_num=2, head_dim=64, v_head_dim=64) + ) + self.assertEqual(window.window, 128) + + def test_a_nextn_head_of_a_linear_attention_trunk_owns_no_state(self): + """The head's config is the trunk's, so it is mamba-ish by class and + lists the trunk's state layers, while the head is a full-attention + block: profiling those layers would decline a draft that fuses.""" + config = _config( + is_hybrid_swa=False, swa_attention_layer_ids=[], num_nextn_predict_layers=1 + ) + with patch( + "sglang.srt.configs.hybrid_arch.mambaish_config", return_value=_TRUNK_STATE + ): + profile = draft_kv_profile(config, num_layers=1, attn_tp_size=1) + self.assertEqual(set(profile.kinds), {LAYER_FULL}) + + def test_a_per_depth_conv_chain_head_owns_one_state_block_per_depth(self): + config = _config( + swa_attention_layer_ids=[1], + num_nextn_predict_layers=2, + hf_text_config=SimpleNamespace(mtp_local_layer_ids=[1]), + ) + with patch( + "sglang.srt.configs.hybrid_arch.mambaish_config", return_value=_TRUNK_STATE + ): + profile = draft_kv_profile(config, num_layers=2, attn_tp_size=1) + self.assertEqual(profile.num_depths, 2) + self.assertEqual(profile.kinds[LAYER_FULL].layer_ids, (0,)) + self.assertEqual(profile.kinds[LAYER_WINDOW].layer_ids, (1,)) + state = profile.kinds[LAYER_STATE] + self.assertEqual(state.layer_ids, (0, 1)) + self.assertEqual(state.geometry, _STATE) + + class TestAsymmetricBackendRule(CustomTestCase): """The rule reads the TARGET's resolved backends and the draft's explicit one (unset inherits the target's); one shared-head_dim backend anywhere @@ -174,6 +433,157 @@ class TestAsymmetricBackendRule(CustomTestCase): ) +class TestHostKindRegistry(CustomTestCase): + """Registration is the extension point: a silent overwrite or a second + primary host would re-route every draft of that kind.""" + + def _dense(self, **overrides): + fields = dict(name="aux", serves=LAYER_WINDOW, family="dense") + fields.update(overrides) + return DenseHostKind(**fields) + + def test_a_duplicate_host_name_is_refused(self): + with _dense_only_registry(), self.assertRaises(AssertionError): + register_host_kind(self._dense(name="full")) + + def test_one_primary_host_per_layer_kind(self): + with _dense_only_registry(), self.assertRaises(AssertionError): + register_host_kind(self._dense(serves=LAYER_FULL)) + + def test_a_fallback_host_must_be_registered_first(self): + with _dense_only_registry(), self.assertRaises(AssertionError): + register_host_kind(self._dense(fallback_host="ghost")) + + def test_hosts_follow_registration_order(self): + """Binders and the boot log walk `hosts()`; dict order of a placement's + regions must not leak into it.""" + region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + with _dense_only_registry(): + register_host_kind(self._dense()) + placement = FusedDraftPlacement.from_counts( + counts={"aux": [1], "full": [1]}, + regions={"aux": region, "full": region}, + ) + self.assertEqual(placement.hosts(), ("full", "aux")) + + def test_describe_names_the_dense_geometry(self): + """Log tooling keys on this line; it is part of the boot contract.""" + region = DenseDraftRegion( + lane_num=2, head_num=1, head_dim=128, v_head_dim=64, store_dtype=_DTYPE + ) + self.assertEqual( + FULL_HOST.describe(region=region, lanes=[(0,), (), (1,)]), + "fused draft region in 'full': 2 lane(s) x 1 kv head(s) x 128/64 k/v " + "head_dim @ torch.bfloat16 = 768 B/token; runner lanes [(0,), (), (1,)]", + ) + + +class TestFusedDraftPlacement(CustomTestCase): + def test_runner_ranges_must_tile_the_region(self): + region = DenseDraftRegion( + lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + with self.assertRaises(AssertionError): + FusedDraftPlacement( + runners=(RunnerLanes(ranges={"full": (0, 1)}),), + regions={"full": region}, + ) + with self.assertRaises(AssertionError): + FusedDraftPlacement(runners=(RunnerLanes(ranges={"full": (0, 1)}),)) + with self.assertRaises(AssertionError): + FusedDraftPlacement.from_counts(counts={"full": [1]}, regions={}) + + def test_a_region_needs_a_registered_host(self): + region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE + ) + with self.assertRaises(AssertionError): + FusedDraftPlacement.from_counts( + counts={"ghost": [1]}, regions={"ghost": region} + ) + + +class TestBootLogReportsEveryPlacedHost(CustomTestCase): + """Every placed host reports through its kind's `describe`, so a host kind + that forgets one dies at boot instead of hiding a placement, and each kind + names its OWN geometry: a shared line would drop to the two fields the + kinds have in common and stop saying what was placed.""" + + def _resolve(self, placement): + cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + with self.assertLogs( + "sglang.srt.mem_cache.kv_cache_configurator", level="INFO" + ) as captured: + out = cfg._fused_draft_for_pool_factory( + FusedDraftDecision(placement=placement) + ) + self.assertIs(out, placement) + return "\n".join(captured.output) + + def test_dense_only_placement_reports_its_region(self): + joined = self._resolve( + FusedDraftPlacement.from_counts( + counts={"full": [1]}, + regions={ + "full": DenseDraftRegion( + lane_num=1, head_num=2, head_dim=16, store_dtype=_DTYPE + ) + }, + ) + ) + self.assertIn("fused draft region in 'full'", joined) + self.assertIn("runner lanes [(0,)]", joined) + self.assertNotIn("fused draft state", joined) + + def test_mixed_placement_reports_each_kind_with_its_own_geometry(self): + joined = self._resolve( + FusedDraftPlacement.from_counts( + counts={"full": [1], "mamba": [1]}, + regions={ + "full": DenseDraftRegion( + lane_num=1, head_num=2, head_dim=16, store_dtype=_DTYPE + ), + "mamba": DraftStateRegion(lane_num=1, state=_STATE), + }, + ) + ) + self.assertIn("fused draft region in 'full'", joined) + self.assertIn( + "fused draft state in 'mamba': 1 lane(s) x 2 conv stream(s)", joined + ) + + +class TestMambaHostPricing(CustomTestCase): + """The mamba sub-pool's fused entry is priced from THIS runner's state + layers, as the mamba factory slices them; the whole model's list would + also count other pipeline ranks'.""" + + def test_a_mamba_region_prices_this_runners_state_layers(self): + cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + cfg.layer_info = SimpleNamespace(start_layer=2, end_layer=4) + cfg.mambaish_config = SimpleNamespace( + mamba2_cache_params=SimpleNamespace( + layers=[0, 1, 2, 3], + shape=SimpleNamespace(conv=[(2, 2)], temporal=(2, 2)), + dtype=SimpleNamespace(conv=_DTYPE, temporal=_DTYPE), + ) + ) + state = DraftStateRegion( + lane_num=1, + state=DraftStateGeometry( + conv_state_shapes=((2, 2),), + conv_dtype=_DTYPE, + temporal_state_shape=(2, 2), + temporal_dtype=_DTYPE, + ), + ) + spec = cfg._mamba_host_spec(state) + self.assertEqual(spec.layer_num, 2) + self.assertIs(spec.draft_region, state) + + class TestUnifiedSWAHeadGeometry(CustomTestCase): """BUG REGRESSION. The hybrid-SWA factories hard-coded a symmetric shape on GPU hosts, so an asymmetric model (MiMo-V2-Flash, 192/128) allocated @@ -185,7 +595,9 @@ class TestUnifiedSWAHeadGeometry(CustomTestCase): cfg.is_hybrid_swa_compress = False cfg.use_mla_backend = False cfg.kv_cache_dtype = _DTYPE - cfg.layer_info = SimpleNamespace(full_attention_layer_ids=[0, 3]) + cfg.layer_info = SimpleNamespace( + full_attention_layer_ids=[0, 3], swa_attention_layer_ids=[1, 2] + ) cfg.model_config = SimpleNamespace( get_num_kv_heads=lambda tp, dcp: 4, head_dim=192, @@ -193,6 +605,8 @@ class TestUnifiedSWAHeadGeometry(CustomTestCase): get_swa_num_kv_heads=lambda tp: 8, swa_head_dim=192, swa_v_head_dim=128, + # The whole model's split; a runner prices its own layer_info. + swa_attention_layer_ids=[1, 2, 4, 5], ) return cfg @@ -214,6 +628,17 @@ class TestUnifiedSWAHeadGeometry(CustomTestCase): (spec.layer_num, spec.head_dim, spec.v_head_dim), (2, 192, 128) ) + def test_the_swa_fused_price_reads_this_runners_window_layers(self): + region = DenseDraftRegion( + lane_num=1, head_num=4, head_dim=64, store_dtype=_DTYPE + ) + with get_parallel().override(attn_tp_size=1, attn_dcp_size=1): + spec = self._configurator()._swa_host_spec(region) + self.assertEqual( + (spec.layer_num, spec.head_num, spec.head_dim, spec.v_head_dim), + (2, 8, 192, 128), + ) + class TestFusedDraftDecision(CustomTestCase): """The target's boot decision over a host whose full sub-pool fuses.""" @@ -244,7 +669,9 @@ class TestFusedDraftDecision(CustomTestCase): is_eagle=lambda: algorithm == "EAGLE", is_dflash_family=lambda: algorithm in ("DFLASH", "DSPARK"), ) - cfg.model_config = SimpleNamespace(is_multi_layer_eagle=False) + cfg.model_config = SimpleNamespace( + is_multi_layer_eagle=False, sliding_window_size=None + ) cfg.kv_cache_dtype = host_kv_dtype cfg.spec_aux_config = SimpleNamespace( eagle_draft_num_layers=1, diff --git a/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py index 4b9a0b7c91..a095da6b5a 100644 --- a/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py +++ b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py @@ -20,37 +20,67 @@ arm, sized by the full sub-allocator's VIRTUAL id space (`max_slots - 1`), not `size_full`, the smaller static token budget: sizing by that would put verify-window writes at high virtual ids out of bounds. +BUG REGRESSION (fused window reach). A multi-layer MTP draft whose window +layers live in the target's swa sub-pool starts its widened draft extend +`front` rows below the committed length, so it reads that many tokens further +back than the target. SWA eviction kept only the target's window and freed +them, and the draft read the sink page instead (NaN under a poisoned pool). +Eviction must keep the draft's reach; every other SWA rule keeps the window. + python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v """ import unittest +from array import array from types import SimpleNamespace from unittest.mock import patch import torch +from sglang.srt.environ import envs +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.mem_cache import kv_cache_configurator as kcc from sglang.srt.mem_cache import unified_draft_pool +from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, ) +from sglang.srt.mem_cache.base_prefix_cache import DecLockRefParams +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.common import checkpoint_kv_cache from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, FusedDraftPlacement, ) -from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool +from sglang.srt.mem_cache.unified_cache.components import ComponentType +from sglang.srt.mem_cache.unified_draft_pool import ( + UnifiedDraftKVPool, + UnifiedDraftSWAKVPool, +) from sglang.srt.mem_cache.unified_memory_pool import ( MHASubPoolSpec, UnifiedKVPool, _store_dtype_for, ) +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel -register_cpu_ci(est_time=5, stage="weekly", runner_config="cpu") +register_cpu_ci(est_time=35, stage="weekly", runner_config="cpu") +maybe_stub_sgl_kernel() + +from sglang.srt.speculative.multi_layer_eagle_worker_v2 import ( + MultiLayerEagleDraftWorker, + MultiLayerEagleWorkerV2, +) class _FakeKVCache: @@ -83,6 +113,40 @@ class _CapturedSizes(Exception): _PS = 2 +def _bare(cls, **attrs): + # Skip __init__: only the attributes the code under test reads are set. + obj = object.__new__(cls) + for name, value in attrs.items(): + setattr(obj, name, value) + return obj + + +def _multi_layer_worker(pools_and_windows, *, front): + """A multi-layer MTP worker whose draft runners hold these (pool, window) + pairs and whose draft extend is widened by `front` rows.""" + runners = [ + SimpleNamespace(token_to_kv_pool=pool, sliding_window_size=window) + for pool, window in pools_and_windows + ] + draft = _bare( + MultiLayerEagleDraftWorker, + draft_runner_list=runners, + draft_extend_num_front_tokens=front, + ) + return _bare(MultiLayerEagleWorkerV2, _draft_worker=draft) + + +def _swa_cache_params(*, sliding_window_size, draft_swa_window): + return CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + page_size=1, + sliding_window_size=sliding_window_size, + draft_swa_window=draft_swa_window, + ) + + class TestDraftBindingDispatch(CustomTestCase): def setUp(self): # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived @@ -98,6 +162,7 @@ class TestDraftBindingDispatch(CustomTestCase): n_full=32, n_swa=16, kv_dtype=torch.bfloat16, + window_draft=False, ): store_dtype = _store_dtype_for(kv_dtype) region = ( @@ -111,6 +176,8 @@ class TestDraftBindingDispatch(CustomTestCase): if with_draft_region else None ) + # A window draft's region rides in the swa sub-pool instead. + host = "swa" if window_draft else "full" full_spec = MHASubPoolSpec( name="full", layer_num=2, @@ -118,7 +185,7 @@ class TestDraftBindingDispatch(CustomTestCase): head_dim=8, store_dtype=store_dtype, grow_direction="down", - draft_region=region, + draft_region=region if host == "full" else None, ) swa_spec = MHASubPoolSpec( name="swa", @@ -127,6 +194,7 @@ class TestDraftBindingDispatch(CustomTestCase): head_dim=8, store_dtype=store_dtype, grow_direction="up", + draft_region=region if host == "swa" else None, ) total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes() pool = UnifiedKVPool( @@ -136,7 +204,9 @@ class TestDraftBindingDispatch(CustomTestCase): enable_memory_saver=False, page_size=_PS, fused_draft=( - FusedDraftPlacement(region=region, runner_lane_counts=(1,)) + FusedDraftPlacement.from_counts( + counts={host: [1]}, regions={host: region} + ) if region is not None else None ), @@ -302,6 +372,247 @@ class TestDraftBindingDispatch(CustomTestCase): self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) self.assertEqual(pools.req_to_token_pool.kind, "private-compact") + def test_window_draft_binds_the_swa_composite(self): + """MiMoV2MTP shape: the runner's one layer is a window layer placed in + the swa sub-pool, so it binds the SWA-shaped composite (the backends + key their swa rail on it), with no full side at all.""" + alloc = self._swa_allocator(with_draft_region=True, window_draft=True) + with patch.object(unified_draft_pool, "draft_swa_layer_ids", return_value=(0,)): + pools = self._run( + algorithm=SpeculativeAlgorithm.EAGLE3, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + pool = pools.token_to_kv_pool + self.assertIsInstance(pool, UnifiedDraftSWAKVPool) + self.assertEqual(pool.layers_mapping, {0: (0, True)}) + self.assertIsNone(pool.full_kv_pool) + self.assertIs(pools.token_to_kv_pool_allocator, alloc) + + def test_a_per_depth_window_runner_trusts_the_placement(self): + """BUG REGRESSION. `adjust_hybrid_swa_layer_ids` clips the draft + config's window list to the runner's own block range, so a per-depth + head above the first block reports NO window layer. The target placed + that runner in the swa sub-pool from the pristine config, so a binder + re-deriving the kind from the clipped config bound the wrong side and + died on the lane-count assert (an Inkling MTP head, depth 2). + A runner holding only swa lanes binds swa, whatever the config says.""" + alloc = self._swa_allocator(with_draft_region=True, window_draft=True) + # The clipped config: this runner's layer is NOT listed as a window layer. + with patch.object(unified_draft_pool, "draft_swa_layer_ids", return_value=()): + pools = self._run( + algorithm=SpeculativeAlgorithm.EAGLE3, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + pool = pools.token_to_kv_pool + self.assertIsInstance(pool, UnifiedDraftSWAKVPool) + self.assertEqual(pool.layers_mapping, {0: (0, True)}) + self.assertIsNone(pool.full_kv_pool) + + def _bound_draft_pool(self, *, window_draft: bool): + alloc = self._swa_allocator(with_draft_region=True, window_draft=window_draft) + window_ids = (0,) if window_draft else () + with patch.object( + unified_draft_pool, "draft_swa_layer_ids", return_value=window_ids + ): + pools = self._run( + algorithm=SpeculativeAlgorithm.EAGLE, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + return pools.token_to_kv_pool + + def test_a_widened_window_draft_widens_the_swa_retain_window(self): + """Inkling-Small's shape: three depths bound to the swa sub-pool at + window 511 and a draft extend widened by 5 front rows. The draft reads + 516 tokens back, so eviction keeps 516 while the window stays 511.""" + pool = self._bound_draft_pool(window_draft=True) + self.assertIsInstance(pool, UnifiedDraftSWAKVPool) + worker = _multi_layer_worker([(pool, 511)] * 3, front=5) + self.assertEqual(worker.draft_swa_window, 516) + params = _swa_cache_params( + sliding_window_size=511, draft_swa_window=worker.draft_swa_window + ) + self.assertEqual(params.swa_retain_window, 516) + self.assertEqual(params.sliding_window_size, 511) + + def test_a_draft_outside_the_swa_sub_pool_keeps_the_window(self): + """Only a widened draft in the swa sub-pool reads past the window: one + folded into the full sub-pool, a private pool (the baseline), or an + extend without front rows leaves the retain window at the target's.""" + full_pool = self._bound_draft_pool(window_draft=False) + self.assertIsInstance(full_pool, UnifiedDraftKVPool) + cases = [ + (full_pool, 5), + (object.__new__(SWAKVPool), 5), + (self._bound_draft_pool(window_draft=True), 0), + ] + for pool, front in cases: + worker = _multi_layer_worker([(pool, 511)], front=front) + params = _swa_cache_params( + sliding_window_size=511, draft_swa_window=worker.draft_swa_window + ) + self.assertEqual(params.swa_retain_window, 511, type(pool).__name__) + + +_WINDOW = 4 +# A window-4 draft whose extend starts 5 rows below the committed length. +_DRAFT_REACH = 9 + + +class TestSWARetainWindow(CustomTestCase): + """BUG REGRESSION. Both frees of a running request's SWA, decode eviction + and the free at a checkpoint insert, must keep what a fused draft reads + (9 tokens back here), not only the target's window (4).""" + + def setUp(self): + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + def _fixture(self, cache_cls, *, draft_swa_window, num_tokens=40, disable=False): + req_to_token_pool = ReqToTokenPool( + size=4, max_context_len=64, device="cpu", enable_memory_saver=False + ) + kv_pool = SWAKVPool( + size=64, + size_swa=64, + page_size=1, + dtype=torch.bfloat16, + head_num=1, + head_dim=8, + swa_attention_layer_ids=[0], + full_attention_layer_ids=[1], + device="cpu", + ) + allocator = SWATokenToKVPoolAllocator( + size=64, + size_swa=64, + page_size=1, + dtype=torch.bfloat16, + device="cpu", + kvcache=kv_pool, + need_sort=False, + ) + cache = cache_cls( + CacheInitParams( + disable=disable, + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + page_size=1, + sliding_window_size=_WINDOW, + draft_swa_window=draft_swa_window, + tree_components=(ComponentType.FULL, ComponentType.SWA), + ) + ) + req = Req( + rid="retain-window", + origin_input_text="", + origin_input_ids=array("q"), + sampling_params=SamplingParams(temperature=0, max_new_tokens=1), + ) + req_to_token_pool.alloc([req]) + req.origin_input_ids = list(range(1, num_tokens + 1)) + req.output_ids = [] + kv_indices = allocator.alloc(num_tokens) + req_to_token_pool.write((req.kv.req_pool_idx, slice(0, num_tokens)), kv_indices) + batch = _bare( + ScheduleBatch, + reqs=[req], + tree_cache=cache, + token_to_kv_pool_allocator=allocator, + req_to_token_pool=req_to_token_pool, + forward_mode=ForwardMode.DECODE, + ) + return batch, req, kv_indices + + def _decode(self, cache_cls, *, draft_swa_window, disable=False): + """Evict at every committed length (the worst case for the draft) and + record the frontier and whether each slot the draft reads is mapped.""" + batch, req, kv_indices = self._fixture( + cache_cls, draft_swa_window=draft_swa_window, disable=disable + ) + mapping = batch.token_to_kv_pool_allocator.full_to_swa_index_mapping + req.decode_batch_idx = 1 + steps = [] + with envs.SGLANG_SWA_EVICTION_INTERVAL.override(1): + for committed in range(12, 40): + req.origin_input_ids = list(range(1, committed + 2)) + batch.maybe_evict_swa() + read = kv_indices[committed - _DRAFT_REACH : committed] + steps.append( + ( + committed, + req.kv.get_evicted_seqlen(ComponentType.SWA), + bool((mapping[read] > 0).all()), + ) + ) + return steps + + def test_decode_eviction_keeps_every_slot_the_draft_reads(self): + # Radix cache on, and off (the disabled tree serves --disable-radix-cache). + for disable in (False, True): + with self.subTest(disable=disable): + for committed, frontier, mapped in self._decode( + UnifiedRadixCache, draft_swa_window=_DRAFT_REACH, disable=disable + ): + self.assertEqual(frontier, committed - _DRAFT_REACH) + self.assertTrue(mapped, committed) + # The target's window alone frees 5 slots the draft still reads. + for committed, frontier, mapped in self._decode( + UnifiedRadixCache, draft_swa_window=0, disable=disable + ): + self.assertEqual(frontier, committed - _WINDOW) + self.assertFalse(mapped, committed) + # maybe_evict_swa reads the retain window off every SWA cache. + batch, _, _ = self._fixture(PureSWARadixCache, draft_swa_window=0) + self.assertEqual(batch.tree_cache.swa_retain_window, _WINDOW) + + def test_the_leaf_lock_outlives_the_draft_reach(self): + """With the prefix lock released after the window, it must wait for + the draft's reach: until then the draft still reads the prefix.""" + for draft_swa_window, released_at in ((_DRAFT_REACH, 9), (0, 4)): + batch, req, _ = self._fixture( + UnifiedRadixCache, draft_swa_window=draft_swa_window + ) + req.lock = SimpleNamespace( + receipt=DecLockRefParams(component_lock_uuids={ComponentType.SWA: 1}) + ) + with ( + envs.SGLANG_OPT_RELEASE_PREFILL_SWA.override(True), + patch.object(batch.tree_cache, "release_swa") as release, + ): + for step in range(1, 13): + req.decode_batch_idx = step + batch.maybe_evict_swa() + if release.called: + break + self.assertEqual(step, released_at) + + def test_the_free_at_a_checkpoint_keeps_every_slot_the_draft_reads(self): + pre_len = 20 + for draft_swa_window, frontier in ( + (_DRAFT_REACH, pre_len - 1 - _DRAFT_REACH), + (0, pre_len - 1 - _WINDOW), + ): + batch, req, _ = self._fixture( + UnifiedRadixCache, draft_swa_window=draft_swa_window, num_tokens=pre_len + ) + cache = batch.tree_cache + req.full_untruncated_fill_ids = array("q", req.origin_input_ids) + req.extend_end = pre_len + req.kv.kv_committed_len = pre_len + req.kv.cache_protected_len = 0 + req.last_node = cache.root_node_handle() + req.lock = None + req.extra_key = None + with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True): + checkpoint_kv_cache(req, cache) + self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), frontier) + cache.unlock(req.lock) + cache.sanity_check() + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py index 2e1d1a7c67..3ef24b1569 100644 --- a/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py +++ b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py @@ -16,7 +16,8 @@ A FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to the target's allocator) translates exactly as the target does -- same v2p table, same physical ids, the draft parts sitting inside the host entries -- and, routing no window -layers, has no sliding-window write ids. A PRIVATE-POOL draft (own +layers, has no sliding-window write ids; one with window layers in the swa +sub-pool rides the target's swa rail. A PRIVATE-POOL draft (own virtual-indexed buffer) stays a strict passthrough. The draft writers outside the draft's forward (DFLASH/DSPARK's target-hidden writes, the multi-step draft decode's kv indices) read the same ids. @@ -38,13 +39,16 @@ from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, FusedDraftPlacement, ) -from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.unified_draft_pool import ( + UnifiedDraftKVPool, + UnifiedDraftSWAKVPool, +) from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=6, suite="base-a-test-cpu") +register_cpu_ci(est_time=10, suite="base-a-test-cpu") _DEV = "cpu" _PS = 2 @@ -73,7 +77,7 @@ class _FakeUnifiedSWAKVPool: self._swa_allocator = swa_allocator -def _build(n_full=32, n_swa=16): +def _build(n_full=32, n_swa=16, *, swa_region=None): full_spec = MHASubPoolSpec( name="full", layer_num=2, @@ -92,17 +96,21 @@ def _build(n_full=32, n_swa=16): head_dim=4, store_dtype=torch.bfloat16, grow_direction="up", + draft_region=swa_region, ) total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes() + counts = {"full": [1]} + regions = {"full": full_spec.draft_region} + if swa_region is not None: + counts["swa"] = [1] + regions["swa"] = swa_region pool = UnifiedKVPool( total_bytes=total, sub_pool_specs=[full_spec, swa_spec], device=_DEV, enable_memory_saver=False, page_size=_PS, - fused_draft=FusedDraftPlacement( - region=full_spec.draft_region, runner_lane_counts=(1,) - ), + fused_draft=FusedDraftPlacement.from_counts(counts=counts, regions=regions), ) kvcache = _FakeUnifiedSWAKVPool(pool) allocator = UnifiedSWATokenToKVPoolAllocator( @@ -164,6 +172,135 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): # A dense draft pool routes no window layers: no swa write loc. self.assertIsNone(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW)) + def test_swa_shaped_fused_draft_rides_the_target_swa_rail(self): + """A draft with window layers binds the swa sub-pool, so its runner + gets the same swa rail as the target: the window write ids and the + window read table are the target's, byte for byte.""" + swa_region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16 + ) + pool, allocator, kvcache, _ = _build(swa_region=swa_region) + composite = UnifiedDraftSWAKVPool( + unified_buffer=pool, + host_allocator=allocator, + page_size=_PS, + full_layer_lanes={0: 0}, + swa_layer_lanes={1: 0}, + ) + v = allocator.alloc(2 * _PS) + self.assertIsNotNone(v) + rt = torch.zeros((4, 16), dtype=torch.int32) + rt[0, : v.numel()] = v.to(torch.int32) + views = [] + for pool_obj in (composite, kvcache): + translator = KVIndexTranslator( + req_to_token=rt, + token_to_kv_pool_allocator=allocator, + token_to_kv_pool=pool_obj, + page_size=_PS, + device=_DEV, + ) + self.assertTrue(translator.is_translating) + fb = SimpleNamespace( + out_cache_loc=v, + req_pool_indices=torch.tensor([0], dtype=torch.int64), + seq_lens=torch.tensor([2 * _PS], dtype=torch.int64), + seq_lens_cpu=torch.tensor([2 * _PS], dtype=torch.int64), + ) + translator.bind_own_plan(fb) + views.append( + ( + translator.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), + translator.read_table( + fb.kv_loc_plan, kind=IdSpaceKind.SLIDING_WINDOW, rows=1 + ).ids, + ) + ) + (draft_write, draft_read), (target_write, target_read) = views + self.assertTrue( + torch.equal(draft_write, allocator.translate_loc_from_full_to_swa(v)) + ) + self.assertTrue(torch.equal(draft_write, target_write)) + self.assertTrue(torch.equal(draft_read, target_read)) + + def test_a_window_depth_reads_the_plan_a_dense_depth_built(self): + """A multi-layer draft plans each draft extend once, through its first + depth's runner, and every depth reads that plan. When the first depth + is dense and a later one has window layers, the later one reads its + window ids from a plan whose runner routes no window layers.""" + swa_region = DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16 + ) + pool, allocator, kvcache, dense_pool = _build(swa_region=swa_region) + window_pool = UnifiedDraftSWAKVPool( + unified_buffer=pool, + host_allocator=allocator, + page_size=_PS, + full_layer_lanes={}, + swa_layer_lanes={1: 0}, + ) + v = allocator.alloc(2 * _PS) + self.assertIsNotNone(v) + rt = torch.zeros((4, 16), dtype=torch.int32) + rt[0, : v.numel()] = v.to(torch.int32) + dense, window, target = ( + KVIndexTranslator( + req_to_token=rt, + token_to_kv_pool_allocator=allocator, + token_to_kv_pool=pool_obj, + page_size=_PS, + device=_DEV, + ) + for pool_obj in (dense_pool, window_pool, kvcache) + ) + self.assertIsNone(dense.space(IdSpaceKind.SLIDING_WINDOW)) + + def batch(): + return SimpleNamespace( + out_cache_loc=v, + req_pool_indices=torch.tensor([0], dtype=torch.int64), + seq_lens=torch.tensor([2 * _PS], dtype=torch.int64), + seq_lens_cpu=torch.tensor([2 * _PS], dtype=torch.int64), + ) + + def window_ids(translator, plan): + # The Triton backend's sliding-window buffer fill. + out = torch.full((2 * _PS,), -1, dtype=torch.int64) + translator.pack_read_stream( + plan, + req_pool_indices=plan.req_pool_indices, + seq_lens=plan.seq_lens, + indptr=torch.tensor([0, 2 * _PS], dtype=torch.int32), + out=out, + kind=IdSpaceKind.SLIDING_WINDOW, + ) + return out + + dense_fb, target_fb = batch(), batch() + dense.bind_own_plan(dense_fb) + target.bind_own_plan(target_fb) + expected = allocator.translate_loc_from_full_to_swa(v) + self.assertTrue(torch.equal(window_ids(window, dense_fb.kv_loc_plan), expected)) + self.assertTrue( + torch.equal(window_ids(target, target_fb.kv_loc_plan), expected) + ) + # A captured graph's window table, which the depth's later gathers + # then read from the plan. + table = window.read_table( + dense_fb.kv_loc_plan, kind=IdSpaceKind.SLIDING_WINDOW, rows=1 + ) + target_table = target.read_table( + target_fb.kv_loc_plan, kind=IdSpaceKind.SLIDING_WINDOW, rows=1 + ) + self.assertTrue(torch.equal(table.ids, target_table.ids)) + src = window.read_source( + dense_fb.kv_loc_plan, + req_pool_indices=dense_fb.req_pool_indices, + bs=1, + kind=IdSpaceKind.SLIDING_WINDOW, + ) + self.assertIs(src.ids, table.ids) + def test_target_hidden_writers_write_physical_ids(self): """DFLASH and DSPARK project the target's hidden states straight into the draft pool. Under fusion that pool takes the target's physical ids: @@ -299,10 +436,10 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): self.assertIsNone(passthrough.full_v2p_table) def test_multi_step_containers_read_the_plans_table(self): - """Every `generate_draft_decode_kv_indices` launch must gather from - `read_source` with its entry granularity. A launch over raw - req_to_token emits VIRTUAL ids, which the fused draft pool cannot - address -- silent garbage drafts.""" + """Every `generate_draft_decode_kv_indices` launch, and every launch of + its sliding-window sibling, must gather from `read_source` with its + entry granularity. A launch over raw req_to_token emits VIRTUAL ids, + which the fused draft pool cannot address -- silent garbage drafts.""" import pathlib import re @@ -312,7 +449,9 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): launching = {} for path in sorted(root.glob("*.py")): text = path.read_text() - launches = len(re.findall(r"generate_draft_decode_kv_indices\[", text)) + launches = len( + re.findall(r"generate_draft_decode_(?:window_)?kv_indices\[", text) + ) if launches: launching[path.name] = ( launches, diff --git a/test/registered/unit/mem_cache/test_kv_loc_plan.py b/test/registered/unit/mem_cache/test_kv_loc_plan.py index e093a10336..6712eff160 100644 --- a/test/registered/unit/mem_cache/test_kv_loc_plan.py +++ b/test/registered/unit/mem_cache/test_kv_loc_plan.py @@ -161,8 +161,8 @@ class TestKVLocPlan(unittest.TestCase): device=_DEV, enable_memory_saver=False, page_size=_PS, - fused_draft=FusedDraftPlacement( - region=full_spec.draft_region, runner_lane_counts=(1,) + fused_draft=FusedDraftPlacement.from_counts( + counts={"full": [1]}, regions={"full": full_spec.draft_region} ), ) kvcache = _FakeUnifiedSWAKVPool(pool) diff --git a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py index af4f71b86b..99ba65a547 100644 --- a/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py +++ b/test/registered/unit/mem_cache/test_mamba_donated_alloc_ratio.py @@ -309,16 +309,12 @@ class TestPPMambaPoolSizing(unittest.TestCase): BUDGET_GB = 8.0 @classmethod - def _pool_size(cls, pp_rank, pp_size): - from sglang.srt import runtime_context as rc + def _cache_params(cls): from sglang.srt.configs.mamba_utils import ( Mamba2CacheParams, Mamba2StateDType, Mamba2StateShape, ) - from sglang.srt.distributed.utils import get_pp_indices - from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator - from sglang.srt.runtime_context import get_schedule shape = Mamba2StateShape( conv=[(4096, 3)], @@ -332,15 +328,33 @@ class TestPPMambaPoolSizing(unittest.TestCase): conv_kernel=0, num_k_heads_per_tp=8, ) - params = Mamba2CacheParams( + return Mamba2CacheParams( shape=shape, dtype=Mamba2StateDType(conv=torch.bfloat16, temporal=torch.float32), layers=list(cls.MAMBA_LAYERS), ) + + @classmethod + def _mamba_budget_bytes(cls): + """The byte budget the solve hands the state pool: the mamba share of + the rest-memory, at the fixture's mamba_full_memory_ratio of 0.5.""" + return cls.BUDGET_GB * (0.5 / 1.5) * (1 << 30) + + @classmethod + def _pool_size(cls, pp_rank, pp_size, fused_state_entry=None): + from sglang.srt import runtime_context as rc + from sglang.srt.distributed.utils import get_pp_indices + from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator + from sglang.srt.runtime_context import get_schedule + + params = cls._cache_params() start, end = get_pp_indices(cls.TOTAL_LAYERS, pp_rank, pp_size) fake = SimpleNamespace( mambaish_config=SimpleNamespace(mamba2_cache_params=params), extra_mamba_cache_bytes_per_req=0, + fused_entry_bytes=lambda name: ( + fused_state_entry if name == "mamba" else None + ), server_args=SimpleNamespace(), spec_algorithm=SimpleNamespace(is_none=lambda: True), layer_info=SimpleNamespace(start_layer=start, end_layer=end), @@ -361,6 +375,27 @@ class TestPPMambaPoolSizing(unittest.TestCase): KVCacheConfigurator._handle_max_mamba_cache(fake, cls.BUDGET_GB) return get_schedule().max_mamba_cache_size + def test_a_fused_state_entry_is_priced_per_slot(self): + """With a draft state block fused into every state slot, the unified + factory carves out K x the FUSED entry; the solve must size K from + that entry, or the buffer over-commits by K x (fused - host) bytes. + + The fused entry is the host's block plus the draft's, so it is always + WIDER than the per-request cost the unfused solve prices -- hence + fewer slots, never more. Deriving it from `mamba_cache_per_req` keeps + that ordering true; a hard-coded constant silently inverted it. + """ + host_per_req = self._cache_params().mamba_cache_per_req + fused_entry = host_per_req + host_per_req // 4 # host + a draft block + budget_bytes = self._mamba_budget_bytes() + + unfused = self._pool_size(0, 1) + fused = self._pool_size(0, 1, fused_state_entry=fused_entry) + + self.assertEqual(unfused, int((budget_bytes - host_per_req) // host_per_req)) + self.assertEqual(fused, int((budget_bytes - fused_entry) // fused_entry)) + self.assertLess(fused, unfused) + def test_stage_is_not_charged_for_the_whole_model(self): solo = self._pool_size(0, 1) staged = self._pool_size(0, 8) @@ -390,6 +425,7 @@ class TestExtraMambaCacheSizing(unittest.TestCase): ) ), extra_mamba_cache_bytes_per_req=extra_bytes, + fused_entry_bytes=lambda name: None, spec_algorithm=SimpleNamespace(is_none=lambda: draft_tokens is None), attn_dp_size=dp_size, pp_size=pp_size, diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index cbc1f6df36..57a7cb2263 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -8,6 +8,7 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") import unittest from unittest.mock import MagicMock, patch +from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.registry import ( _RADIX_CACHE_REGISTRY, TreeCacheBuildContext, @@ -204,9 +205,14 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): ) from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent - params = MagicMock(sliding_window_size=128, page_size=64) - params.token_to_kv_pool_allocator = MagicMock( - spec=DeepSeekV4HiSparseTokenToKVPoolAllocator + params = CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=MagicMock( + spec=DeepSeekV4HiSparseTokenToKVPoolAllocator + ), + page_size=64, + sliding_window_size=128, ) component = SWAComponent(MagicMock(), params) self.assertEqual(component.full_window_pages, 2) diff --git a/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py index 22f943282b..a585bd1f90 100644 --- a/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_rust_unified_radix_cache_unittest.py @@ -6,7 +6,7 @@ import test_unified_radix_cache_unittest as shared_suite from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=57, stage="base-b", runner_config="1-gpu-small") +register_cuda_ci(est_time=80, stage="base-b", runner_config="1-gpu-small") class RustBackendSuite(unittest.TestSuite): diff --git a/test/registered/unit/mem_cache/test_swa_pool_v_head_dim.py b/test/registered/unit/mem_cache/test_swa_pool_v_head_dim.py index d3fd443af4..57910fff68 100644 --- a/test/registered/unit/mem_cache/test_swa_pool_v_head_dim.py +++ b/test/registered/unit/mem_cache/test_swa_pool_v_head_dim.py @@ -34,7 +34,7 @@ _FULL_V_HEAD_DIM = 8 _SWA_V_HEAD_DIM = 8 # MATCHING: this is what routes Inkling into the branch -def _swa_pool(): +def _swa_pool(*, swa_attention_layer_ids=(0, 2), full_attention_layer_ids=(1, 3)): """Inkling-class layer split: full and SWA layers interleaved, layer 0 NOT a full-attention layer, which is why the backend asks the pool at all.""" return SWAKVPool( @@ -44,8 +44,8 @@ def _swa_pool(): dtype=torch.float16, head_num=2, head_dim=_FULL_V_HEAD_DIM, - swa_attention_layer_ids=[0, 2], - full_attention_layer_ids=[1, 3], + swa_attention_layer_ids=list(swa_attention_layer_ids), + full_attention_layer_ids=list(full_attention_layer_ids), device=_DEV, enable_memory_saver=False, ) @@ -56,6 +56,14 @@ class TestSWAPoolVHeadDim(unittest.TestCase): pool = _swa_pool() self.assertEqual(pool.get_v_head_dim(), _FULL_V_HEAD_DIM) + def test_window_only_pool_answers_from_the_swa_side(self): + """BUG REGRESSION. A sliding-window MTP depth is a window-only block, + so its pool has an EMPTY full side; asking that side for a value + buffer killed the draft backend's construction before a token was + served.""" + pool = _swa_pool(swa_attention_layer_ids=(0,), full_attention_layer_ids=()) + self.assertEqual(pool.get_v_head_dim(), _SWA_V_HEAD_DIM) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_mamba_compaction.py b/test/registered/unit/mem_cache/test_unified_mamba_compaction.py index fa3ac6e8ad..6181291bf2 100644 --- a/test/registered/unit/mem_cache/test_unified_mamba_compaction.py +++ b/test/registered/unit/mem_cache/test_unified_mamba_compaction.py @@ -179,6 +179,19 @@ class TestMambaCompaction(CustomTestCase): ]: self.assertTrue(torch.equal(tensor[:, src], tensor[:, dst])) self.assertTrue(torch.equal(f.ngram.context[src], f.ngram.context[dst])) + state_tensors = [ + *f.kv.mamba_cache.conv, + f.kv.mamba_cache.temporal, + f.short.conv_state, + ] + sources = [t[:, src].clone() for t in state_tensors] + ngram_source = f.ngram.context[src].clone() + f.kv.clear_slots(dst) + for tensor, source in zip(state_tensors, sources): + self.assertEqual(torch.count_nonzero(tensor[:, dst]).item(), 0) + self.assertTrue(torch.equal(tensor[:, src], source)) + self.assertEqual(torch.count_nonzero(f.ngram.context[dst]).item(), 0) + self.assertTrue(torch.equal(f.ngram.context[src], ngram_source)) before_empty = f.pool._raw.clone() f.kv.move_kv_cache(f.ids[:0], f.ids[:0]) self.assertTrue(torch.equal(f.pool._raw, before_empty)) diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index aeed32db9d..ba715f2294 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -125,8 +125,8 @@ from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.mem_cache_utils import finish_req from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=50, suite="stage-b-test-1-gpu-small-amd") +register_cuda_ci(est_time=90, stage="base-b", runner_config="1-gpu-small") +register_amd_ci(est_time=90, suite="stage-b-test-1-gpu-small-amd") # A dedicated test entry point overrides this without changing the process-wide # production backend selection. Direct Python-core tests in this module remain @@ -154,6 +154,7 @@ class CacheConfig: # SWA sliding_window_size: Optional[int] = None + draft_swa_window: int = 0 decoder_swa_bounded_replay: bool = False # Mamba @@ -742,6 +743,7 @@ def build_fixture( page_size=cfg.page_size if tree_page_size is None else tree_page_size, disable=False, sliding_window_size=cfg.sliding_window_size, + draft_swa_window=cfg.draft_swa_window, decoder_swa_bounded_replay=cfg.decoder_swa_bounded_replay, tree_components=cfg.components, enable_mamba_extra_buffer=cfg.enable_mamba_extra_buffer, @@ -11425,6 +11427,105 @@ class TestUnifiedRadixPrefetchCorruption(CustomTestCase): cache.sanity_check() +class TestSWADraftHistory(CustomTestCase): + """A widened draft needs SWA history before the target window. A running + checkpoint must lock that history, and an incomplete cached window must + not be reused even when the target alone could read it. The shared suite + runs these contracts against both Python and Rust TreeCore. + """ + + cfg = CacheConfig( + page_size=1, + components=(ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA), + sliding_window_size=511, + draft_swa_window=516, + is_eagle=True, + kv_size=2048, + max_context_len=2048, + num_layers=2, + full_attention_layer_ids=(0,), + head_num=1, + head_dim=8, + enable_mamba_extra_buffer=True, + ) + + def _request(self, pool): + tokens = array("q", range(1, 1025)) + req = Req( + rid="widened-draft", + origin_input_text="", + origin_input_ids=tokens, + sampling_params=SamplingParams(temperature=0, max_new_tokens=16), + ) + pool.alloc([req]) + req.output_ids = [] + req.full_untruncated_fill_ids = tokens + req.extend_end = len(tokens) + req.kv.kv_committed_len = req.kv.kv_allocated_len = len(tokens) + req.kv.cache_protected_len = 0 + req.kv.mamba_last_track_seqlen = len(tokens) + req.kv.mamba_last_track_idx = 0 + req.kv.mamba_next_track_idx = pool.get_mamba_ping_pong_other_idx(0) + req.lock = None + req.extra_key = None + return req + + def test_checkpoint_protects_widened_draft_history_under_eviction(self): + cache, allocator, pool = build_fixture(self.cfg, mamba_cache_chunk_size=64) + req = self._request(pool) + req.last_node = cache.root_node_handle() + kv = allocator.alloc(1024) + pool.write((req.kv.req_pool_idx, slice(0, 1024)), kv) + cache.checkpoint(req, up_to=1024) + self.addCleanup(lambda: cache.unlock(req.lock)) + + # Mamba disables bigram keys: this checkpoint is exactly at 1024. + self.assertEqual(req.kv.cache_protected_len, 1024) + # A three-depth draft with five front rows and three warmup rows has + # its first real query at 1022. Its window needs key 511, which lies + # before the target-only lock [513, 1024). + needed = kv[1024 - self.cfg.draft_swa_window : 1024] + mapping = allocator.full_to_swa_index_mapping + before = mapping[needed].clone() + self.assertTrue(bool((before > 0).all())) + cache.evict(EvictParams(swa_num_tokens=1)) + torch.testing.assert_close(mapping[needed], before) + match = cache.match_prefix( + MatchPrefixParams(key=RadixKey(req.origin_input_ids)) + ) + self.assertEqual(match.device_prefix_len, 1024) + + # Decode advances past the last recurrent checkpoint. Its complete + # draft history must remain until a newer checkpoint replaces it. + cache.evict_sliding_windows(req, 1040) + torch.testing.assert_close(mapping[needed], before) + cache.sanity_check() + + def test_prefix_match_rejects_target_only_swa_history(self): + for retained, expected_match in ((511, 0), (516, 1024)): + with self.subTest(retained=retained): + cache, allocator, pool = build_fixture( + self.cfg, mamba_cache_chunk_size=64 + ) + req = self._request(pool) + kv = allocator.alloc(1024) + cutoff = 1024 - retained + allocator.free_swa_segment(kv[:cutoff], start_pos=0) + cache.insert( + InsertParams( + key=RadixKey(req.origin_input_ids), + value=kv, + mamba_value=req.kv.mamba_pool_idx.unsqueeze(0), + component_evicted_seqlens={ComponentType.SWA: cutoff}, + ) + ) + result = cache.match_prefix( + MatchPrefixParams(key=RadixKey(req.origin_input_ids)) + ) + self.assertEqual(result.device_prefix_len, expected_match) + cache.sanity_check() + + class TestSWAWindowUnderBigramKey(CustomTestCase): """`checkpoint` has to leave the leaf it inserts holding a full sliding window of live SWA. Otherwise the match that follows the insert diff --git a/test/registered/unit/mem_cache/test_unified_tri_pool.py b/test/registered/unit/mem_cache/test_unified_tri_pool.py index cee8087ae3..22bc05cf7b 100644 --- a/test/registered/unit/mem_cache/test_unified_tri_pool.py +++ b/test/registered/unit/mem_cache/test_unified_tri_pool.py @@ -37,6 +37,8 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import ( from sglang.srt.mem_cache.base_prefix_cache import EvictParams from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, + DraftStateGeometry, + DraftStateRegion, FusedDraftPlacement, ) from sglang.srt.mem_cache.unified_cache.components import ComponentType @@ -55,7 +57,7 @@ from sglang.test.unified_allocator_fixtures import ( # Hermetic convention of this directory's pool tests: plain unittest.TestCase, # only ci_register imported (no heavy sglang.test.test_utils chain). -register_cpu_ci(est_time=8, suite="base-a-test-cpu") +register_cpu_ci(est_time=15, suite="base-a-test-cpu") _DEV = "cpu" @@ -1229,7 +1231,9 @@ class TestTriFactorySizing(unittest.TestCase): region = DenseDraftRegion( lane_num=1, head_num=2, head_dim=4, store_dtype=torch.float16 ) - placement = FusedDraftPlacement(region=region, runner_lane_counts=(1,)) + placement = FusedDraftPlacement.from_counts( + counts={"full": [1]}, regions={"full": region} + ) bundle = init_unified_mamba_swa_pools( **self._factory_kwargs(fused_draft=placement) ) @@ -1240,6 +1244,41 @@ class TestTriFactorySizing(unittest.TestCase): for other in ("swa", "mamba"): self.assertIsNone(pool.spec(other).draft_region, other) + def _state_placement(self): + region = DraftStateRegion( + lane_num=2, + state=DraftStateGeometry( + conv_state_shapes=((3, 8),), + conv_dtype=torch.bfloat16, + temporal_state_shape=(0, 0, 0), + temporal_dtype=torch.float32, + ), + ) + return FusedDraftPlacement.from_counts( + counts={"mamba": [1, 1]}, regions={"mamba": region} + ) + + def test_fused_draft_state_clone_shares_the_request_slot_space(self): + """A draft runner's req pool over the fused state block keeps the + target's request->slot mappings and slot allocator (the draft's slot + ids ARE the target's), and views only its own block.""" + bundle = init_unified_mamba_swa_pools( + **self._factory_kwargs(fused_draft=self._state_placement()) + ) + target = bundle.req_to_token_pool + clone = target.clone_for_fused_draft(layer_lanes={1: 1}) + self.assertIs( + clone.req_index_to_mamba_index_mapping, + target.req_index_to_mamba_index_mapping, + ) + self.assertIs(clone.mamba_allocator, target.mamba_allocator) + self.assertIs(clone.req_to_token, target.req_to_token) + self.assertEqual(clone.mamba_map, {1: 0}) + conv = clone.mamba2_layer_cache(1).conv[0] + self.assertEqual(tuple(conv.shape), (target.mamba_pool.size + 1, 3, 8)) + draft_conv, _ = bundle.unified_memory_pool.build_draft_state_views("mamba") + self.assertEqual(conv.data_ptr(), draft_conv[0][1].data_ptr()) + def test_bs1_floor_fails_loud_before_construction(self): """A budget far below one worst-case request must raise BEFORE any pool construction -- under-sizing is a retract LIVELOCK at runtime.""" diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 37c2c2b1d0..1ee04b6526 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1755,6 +1755,27 @@ class TestFusedDraftPricing(unittest.TestCase): config = cfg.calculate_pool_sizes(available, 1) self._assert_unified_bytes_price_the_fused_entry(cfg, config, available) + def test_fused_swa_entry_replaces_the_swa_terms(self): + """A draft placed in the swa sub-pool is priced through the fused swa + entry: the separate per-layer swa draft term must go, or the draft is + charged twice (once inside the entry, once as a private pool).""" + mr, cfg = self._make(fused_entry=None, draft_layers=2) + mr.spec_aux_config.eagle_draft_swa_num_layers = 2 + mr.fused_entry_bytes.side_effect = lambda name: 4_321 if name == "swa" else None + with mock_cpu_env(): + from sglang.srt.model_executor.pool_configurator import ( + HybridSWAPoolConfigurator, + ) + + fused = HybridSWAPoolConfigurator(mr) + self.assertEqual(fused._draft_swa_layers_num, 0) + self.assertEqual(fused._draft_full_layers_num, 0) + self.assertEqual( + fused._cell_size, + fused._full_per_token * fused._full_layers_num + + fused._swa_full_tokens_ratio * 4_321, + ) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_resolve_sliding_window_size.py b/test/registered/unit/model_executor/test_resolve_sliding_window_size.py new file mode 100644 index 0000000000..00ab5246d2 --- /dev/null +++ b/test/registered/unit/model_executor/test_resolve_sliding_window_size.py @@ -0,0 +1,82 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`resolve_sliding_window_size` and the Inkling MTP head's window convention. + +BUG REGRESSION. The resolver prefers a model's own +`get_attention_sliding_window_size`, and Inkling's trunk answers one less +than its extent. Its MTP head declared no such method, so the resolver fell +through to the config field: the head's backend ran one token wider than +the trunk's window layers, and a banded depth's per-step window rail read +one token further back than the swa sub-pool retains. + +Registered on a CUDA runner, not the CPU gate: `_head` needs the real +`InklingForConditionalGenerationMTP`, whose module pulls in the flash-attn +cute interface and therefore `cutlass`, which a CPU image does not carry. + + python -m pytest test/registered/unit/model_executor/test_resolve_sliding_window_size.py -v +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.model_executor.model_runner_components.load_model_utils import ( + resolve_sliding_window_size, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="extra-a", runner_config="1-gpu-small") + + +def _head(*, depth, local_layer_ids=(1,), local_extent=64, window=128): + """A stand-in for the MTP head: only what the window method reads.""" + from sglang.srt.models.inkling import InklingForConditionalGenerationMTP + + head = SimpleNamespace( + draft_model_idx=depth, + text_config=SimpleNamespace( + mtp_local_layer_ids=list(local_layer_ids), + mtp_local_extent=local_extent, + sliding_window_size=window, + ), + ) + head.get_attention_sliding_window_size = ( + InklingForConditionalGenerationMTP.get_attention_sliding_window_size.__get__( + head + ) + ) + return head + + +class TestInklingMTPWindowConvention(CustomTestCase): + def test_a_banded_depth_runs_one_under_its_own_extent(self): + self.assertEqual(_head(depth=1).get_attention_sliding_window_size(), 63) + + def test_a_full_depth_runs_one_under_the_trunk_window(self): + self.assertEqual(_head(depth=0).get_attention_sliding_window_size(), 127) + + def test_the_resolver_prefers_the_head_over_the_config_field(self): + """The config keeps the raw extent (the swa host admission reads it); + the runner's window must be the head's convention, not that field.""" + model_config = SimpleNamespace( + is_hybrid_swa=True, sliding_window_size=64, attention_chunk_size=None + ) + self.assertEqual(resolve_sliding_window_size(_head(depth=1), model_config), 63) + self.assertEqual( + resolve_sliding_window_size(SimpleNamespace(), model_config), 64 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_spec_utils_reexport_surface.py b/test/registered/unit/spec/test_spec_utils_reexport_surface.py new file mode 100644 index 0000000000..e949a8c185 --- /dev/null +++ b/test/registered/unit/spec/test_spec_utils_reexport_surface.py @@ -0,0 +1,107 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`spec_utils` is a RE-EXPORT hub: the speculative Triton kernels live in +`sglang.kernels.ops.speculative.*` and are forwarded here as `X as X` so the +srt modules import one name from one place. + +BUG REGRESSION. A new kernel was added to `cache_locs` and imported by +`triton_backend` from `spec_utils`, but never forwarded there. Nothing local +caught it: the name is imported and used, so ruff's F401/F821 stay quiet, and +`py_compile` only parses. It surfaced as an ImportError at server boot that +took down EVERY server using the Triton backend, spec-off included, because the +import is at module scope. + +This walks the REAL consumers (an AST scan of srt, not a hand-kept list) and +checks each imported name resolves, so a forgotten forward fails here instead +of on a GPU. + + python -m pytest test/registered/unit/spec/test_spec_utils_reexport_surface.py -v +""" + +import ast +import unittest +from pathlib import Path + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=25, suite="base-a-test-cpu") + +_HUB = "sglang.srt.speculative.spec_utils" + + +def _srt_root() -> Path: + """The installed `sglang/srt` tree. + + Via the PARENT package: `sglang.srt` ships no `__init__.py`, so it is a PEP + 420 namespace package whose `__file__` is None. This is the idiom the other + srt-walking tests use (test_server_args_namespaces.py and friends). + """ + import sglang + + return Path(sglang.__file__).resolve().parent / "srt" + + +def _consumers(): + """(module path, imported names) for every srt module importing from the hub.""" + root = _srt_root() + assert root.is_dir(), f"srt tree not found at {root}" + for path in sorted(root.rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (SyntaxError, UnicodeDecodeError): + continue + for node in ast.walk(tree): + if not isinstance(node, ast.ImportFrom) or node.level: + continue + if node.module == _HUB: + yield path, tuple(a.name for a in node.names if a.name != "*") + + +class TestSpecUtilsReexportSurface(CustomTestCase): + def test_every_imported_name_resolves(self): + import sglang.srt.speculative.spec_utils as hub + + consumers = list(_consumers()) + # A scan that finds nothing would pass vacuously forever. + self.assertGreater(len(consumers), 5, "AST scan found no spec_utils consumers") + + missing = [ + f"{path.name} imports {name!r}" + for path, names in consumers + for name in names + if not hasattr(hub, name) + ] + self.assertEqual(missing, [], f"{_HUB} does not export: {missing}") + + def test_the_forwarded_kernels_are_the_kernel_module_objects(self): + """A forward must alias the kernel itself; a same-named local shim + would pass the attribute check above while shadowing the real one.""" + import sglang.kernels.ops.speculative.cache_locs as cache_locs + import sglang.srt.speculative.spec_utils as hub + + forwarded = [ + a.asname or a.name + for node in ast.walk(ast.parse(Path(hub.__file__).read_text())) + if isinstance(node, ast.ImportFrom) + and node.module == "sglang.kernels.ops.speculative.cache_locs" + for a in node.names + ] + self.assertIn("generate_draft_decode_kv_indices", forwarded) + for name in forwarded: + self.assertIs(getattr(hub, name), getattr(cache_locs, name), name) + + +if __name__ == "__main__": + unittest.main()