diff --git a/python/sglang/kernels/ops/kvcache/kv_read_table.py b/python/sglang/kernels/ops/kvcache/kv_read_table.py index c00201c32e..d4b216b480 100644 --- a/python/sglang/kernels/ops/kvcache/kv_read_table.py +++ b/python/sglang/kernels/ops/kvcache/kv_read_table.py @@ -30,10 +30,11 @@ Two delivery forms over that one formula: which the pool bounds, where a page table's width is bounded only by `max_context_len`. -PREFIX-ONLY per row: nothing past the row's live prefix is written, so a -caller-owned buffer keeps what it had there -- which is what lets a captured -cuda-graph buffer be refreshed in place. Readers bound themselves by -`cache_seqlens` and never look past the prefix. +PREFIX-ONLY per row by default: nothing past the row's live prefix is written, +so a caller-owned buffer keeps what it had there -- which is what lets a +captured cuda-graph buffer be refreshed in place. Readers bound themselves by +`cache_seqlens` and never look past the prefix. A fresh (`torch.empty`) table +asks for the sink past the prefix instead (`zero_tail`). A `-1` in `req_to_token` and a freed (`-1`) v2p row both clamp to entry 0, the reserved padding slot, so a kernel dereferences padding, not a wild address. @@ -60,6 +61,56 @@ _TARGET_BLOCKS = 1024 @triton.jit +def _gather_translate_items( + row_in, # this row of `req_to_token` -- VIRTUAL token ids + row_out, # where this row's items land + v2p_ptr, # [num_pages + 1] int64 -- virtual->physical page table + live, # scalar: the row reads `req_to_token`; else every item is the sink + kv_start, # first token of the row's window + n_items, # live items in the row + n_written, # items stored: `n_items`, or a wider zeroed row + item_begin, # this program's first item + item_stride, # items one program advances per loop trip + PAGE_SIZE: tl.constexpr, + EMIT_PER_TOKEN: tl.constexpr, + OUT_INT64: tl.constexpr, + ZERO_TAIL: tl.constexpr, + BLOCK: tl.constexpr, +): + for start in range(item_begin, n_written, item_stride): + item = start + tl.arange(0, BLOCK) + mask = item < n_items + load_mask = mask & live + pos = kv_start + item + if EMIT_PER_TOKEN: + page = pos // PAGE_SIZE + else: + page = pos + tok = tl.load( + row_in + page.to(tl.int64) * PAGE_SIZE, mask=load_mask, other=0 + ).to(tl.int64) + # Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but + # -1 at ps == 1, which would read one element BEFORE `v2p`. + vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE) + entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=load_mask, other=0), 0) + if EMIT_PER_TOKEN: + value = entry * PAGE_SIZE + pos % PAGE_SIZE + else: + value = entry + store_mask = mask + if ZERO_TAIL: + value = tl.where(mask, value, 0) + store_mask = item < n_written + if OUT_INT64: + tl.store(row_out + item, value, mask=store_mask) + else: + tl.store(row_out + item, value.to(tl.int32), mask=store_mask) + + +# The per-call widths and lengths stay unspecialized: Triton would otherwise +# compile a new variant (`== 1`, `% 16`) mid-serving each time a batch's shape +# hits a combination it has not seen. +@triton.jit(do_not_specialize=["out_stride", "seq_len_delta", "max_row_items"]) def build_kv_read_indices_kernel( req_to_token_ptr, # in: [max_reqs, max_context] -- VIRTUAL token ids req_pool_indices_ptr, # in: [bs] -- req_to_token row per batch lane @@ -76,6 +127,7 @@ def build_kv_read_indices_kernel( PAGE_SIZE: tl.constexpr, EMIT_PER_TOKEN: tl.constexpr, OUT_INT64: tl.constexpr, + ZERO_TAIL: tl.constexpr, BLOCK: tl.constexpr, ): bid = tl.program_id(0) @@ -98,30 +150,24 @@ def build_kv_read_indices_kernel( # A widened window can outrun a fixed-width row (a verify near the # context limit); writing past it would land in the next row. n_items = tl.minimum(n_items, max_row_items) - - for start in range(tl.program_id(1) * BLOCK, n_items, item_stride): - item = start + tl.arange(0, BLOCK) - mask = item < n_items - pos = kv_start + item - if EMIT_PER_TOKEN: - page = pos // PAGE_SIZE - else: - page = pos - tok = tl.load(row_in + page.to(tl.int64) * PAGE_SIZE, mask=mask, other=0).to( - tl.int64 - ) - # Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but - # -1 at ps == 1, which would read one element BEFORE `v2p`. - vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE) - entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=mask, other=0), 0) - if EMIT_PER_TOKEN: - value = entry * PAGE_SIZE + pos % PAGE_SIZE - else: - value = entry - if OUT_INT64: - tl.store(row_out + item, value, mask=mask) - else: - tl.store(row_out + item, value.to(tl.int32), mask=mask) + # A fresh (unzeroed) table also takes the sink past the live prefix. + n_written = max_row_items if ZERO_TAIL else n_items + _gather_translate_items( + row_in, + row_out, + v2p_ptr, + True, + kv_start, + n_items, + n_written, + tl.program_id(1) * BLOCK, + item_stride, + PAGE_SIZE=PAGE_SIZE, + EMIT_PER_TOKEN=EMIT_PER_TOKEN, + OUT_INT64=OUT_INT64, + ZERO_TAIL=ZERO_TAIL, + BLOCK=BLOCK, + ) def _launch( @@ -138,6 +184,7 @@ def _launch( kv_start_idx: Optional[torch.Tensor], emit_per_token: bool, seq_len_delta: int = 0, + zero_tail: bool = False, ) -> None: bs = int(req_pool_indices.numel()) item_programs = min( @@ -159,6 +206,7 @@ def _launch( PAGE_SIZE=page_size, EMIT_PER_TOKEN=emit_per_token, OUT_INT64=out.dtype == torch.int64, + ZERO_TAIL=zero_tail, BLOCK=_BLOCK_ITEMS, num_warps=_NUM_WARPS, ) @@ -188,17 +236,18 @@ def build_kv_read_table( max_pages: int, out: torch.Tensor, seq_len_delta: int = 0, + zero_tail: bool = False, ) -> torch.Tensor: """Fill ``out``'s live prefix with PAGE TABLE entries. - ``out`` is caller-owned (fresh zeros for the eager path, the module's - capture-stable buffer for replay) and only its ``[:bs, :max_pages]`` - region's live prefix is written -- never rebound, never tail-cleared. + ``out`` is caller-owned and only its ``[:bs, :max_pages]`` region is + written: each row's live prefix, and with ``zero_tail`` the sink (0) past + it up to ``max_pages`` -- what a fresh ``torch.empty`` table needs, in the + same launch. Never rebound. - ``seq_len_delta`` widens every row's live prefix -- the whole-sequence - verify contract (draft KV read back from the pool). Every row stops at - ``max_pages``: a widened window that outruns the row is cut there rather - than spilling into the next row, on CUDA as on CPU. + ``seq_len_delta`` widens every row's live prefix (a verify reads its draft + tokens' KV back from the pool); a widened row is cut at ``max_pages`` + rather than spilling into the next row. """ bs = int(req_pool_indices.numel()) assert out.dtype == torch.int32, ( @@ -228,6 +277,8 @@ def build_kv_read_table( v2p=v2p, page_size=page_size, ).to(torch.int32) + if zero_tail: + out[b, live:max_pages] = 0 return out _launch( @@ -243,6 +294,7 @@ def build_kv_read_table( kv_start_idx=None, emit_per_token=False, seq_len_delta=seq_len_delta, + zero_tail=zero_tail, ) return out diff --git a/python/sglang/kernels/ops/speculative/cache_locs.py b/python/sglang/kernels/ops/speculative/cache_locs.py index 4d1f93d261..03c8c241a4 100644 --- a/python/sglang/kernels/ops/speculative/cache_locs.py +++ b/python/sglang/kernels/ops/speculative/cache_locs.py @@ -89,6 +89,29 @@ def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr): return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0)) +@triton.jit +def _load_token_ids( + row_ptr, + pos, + mask, + v2p, + page_size: tl.constexpr, + ENTRY_PAGE_SIZE: tl.constexpr, + TRANSLATE: tl.constexpr, +): + """Token ids at positions ``pos`` of one read-source row: the row's own + entries when token-granular, ``entry * ps + pos % ps`` when page-granular; + with ``TRANSLATE``, virtual ids mapped through ``v2p`` on the way out.""" + if ENTRY_PAGE_SIZE == 1: + ids = tl.load(row_ptr + pos, mask=mask) + else: + entry = tl.load(row_ptr + pos // ENTRY_PAGE_SIZE, mask=mask, other=0) + ids = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE + if TRANSLATE: + ids = _translate_token_ids(ids, v2p, mask, page_size) + return ids + + @triton.jit def generate_draft_decode_kv_indices( req_pool_indices, @@ -97,8 +120,9 @@ def generate_draft_decode_kv_indices( kv_indices, kv_indptr, positions, - v2p, - pool_len: tl.constexpr, + # Runtime, not constexpr: the unified pool's read table is as wide as the + # iteration's longest row, so a constexpr stride would recompile per width. + row_stride, kv_indices_stride: tl.constexpr, kv_indptr_stride: tl.constexpr, bs_upper: tl.constexpr, @@ -108,6 +132,9 @@ def generate_draft_decode_kv_indices( window_size: tl.constexpr = 0, sink_size: tl.constexpr = 0, NUM_STEPS: tl.constexpr = 0, + ENTRY_PAGE_SIZE: tl.constexpr = 1, + # The source's page table (`KVIndexTable.v2p`), applied with TRANSLATE. + v2p=None, TRANSLATE: tl.constexpr = False, ): # window_size > 0 restricts the draft (not the target) to sink_size prefix @@ -151,7 +178,7 @@ def generate_draft_decode_kv_indices( # Update kv_indices kv_offset = cum_seq_len * topk + bid * iters * topk + topk_id * (seq_len_w + iters) kv_ptr = kv_indices + kv_offset - token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * pool_len + token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride num_loop = tl.cdiv(seq_len_w, BLOCK_SIZE) if NUM_STEPS != 0 and blk >= num_loop and blk > 0: @@ -165,9 +192,9 @@ def generate_draft_decode_kv_indices( copy_offset, recent_start + copy_offset - s_eff, ) - data = tl.load(token_pool_ptr + src, mask=mask) - if TRANSLATE: - data = _translate_token_ids(data, v2p, mask, page_size) + data = _load_token_ids( + token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE + ) tl.store(kv_ptr + copy_offset, data, mask=mask) copy_offset += BLOCK_SIZE else: @@ -179,9 +206,9 @@ def generate_draft_decode_kv_indices( copy_offset, recent_start + copy_offset - s_eff, ) - data = tl.load(token_pool_ptr + src, mask=mask) - if TRANSLATE: - data = _translate_token_ids(data, v2p, mask, page_size) + data = _load_token_ids( + token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE + ) tl.store(kv_ptr + copy_offset, data, mask=mask) # Extension entries and kv_indptr belong to token block 0 alone; other @@ -189,12 +216,14 @@ def generate_draft_decode_kv_indices( if blk == 0: extend_offset = tl.arange(0, iter_upper) if page_size == 1 or topk == 1: - extend_data = tl.load( - token_pool_ptr - + seq_len - + topk_id * num_steps - + tl.arange(0, iter_upper), - mask=extend_offset < iters, + extend_data = _load_token_ids( + token_pool_ptr, + seq_len + topk_id * num_steps + tl.arange(0, iter_upper), + extend_offset < iters, + v2p, + page_size, + ENTRY_PAGE_SIZE, + TRANSLATE, ) else: prefix_len = seq_len @@ -208,14 +237,14 @@ def generate_draft_decode_kv_indices( + topk_id * num_new_pages_per_topk * page_size + last_page_len ) - extend_data = tl.load( - token_pool_ptr + start + extend_offset, - mask=extend_offset < iters, - ) - - if TRANSLATE: - extend_data = _translate_token_ids( - extend_data, v2p, extend_offset < iters, page_size + extend_data = _load_token_ids( + token_pool_ptr, + start + extend_offset, + extend_offset < iters, + v2p, + page_size, + ENTRY_PAGE_SIZE, + TRANSLATE, ) tl.store( diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py index ed475e657b..b97077da2f 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py @@ -20,6 +20,9 @@ class RaggedVerifyWindow(msgspec.Struct, frozen=True): positions: torch.Tensor verify_cache_loc: torch.Tensor verify_ids: torch.Tensor + # Each packed lane's flat position in the `[bs, verify_num_draft_tokens]` + # verify window, -1 on a padded lane. + window_index: torch.Tensor class BuildRaggedVerifyWindow: @@ -129,6 +132,9 @@ def build_ragged_verify_window( positions=positions, verify_cache_loc=verify_cache_loc, verify_ids=verify_ids, + window_index=torch.where( + valid, safe_req.to(torch.int64) * verify_num_draft_tokens + within, -1 + ), ) @@ -176,7 +182,7 @@ def build_ragged_verify_window_triton( verify_lens = layout.verify_lens.to(device=device, dtype=torch.int32) padded_total = layout.graph_num_tokens - req_id, within, _valid = compact_row_index_triton( + req_id, within, valid = compact_row_index_triton( verify_lens=verify_lens, padded_total=padded_total, device=device ) real_cache_loc = assign_extend_cache_locs_func( @@ -218,6 +224,11 @@ def build_ragged_verify_window_triton( positions=positions, verify_cache_loc=verify_cache_loc, verify_ids=verify_ids, + window_index=torch.where( + valid, + req_id.clamp(max=bs - 1).to(torch.int64) * verify_num_draft_tokens + within, + -1, + ), ) diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 2041e1e3d5..8c71bc889d 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -499,10 +499,9 @@ def _assert_spec_verify_backends( ) -> None: """Refuse target backends whose verify id rails are not translation-audited. - Only the target's prefill/decode pair is checked here. A draft that fuses - into the target's pages must run on the translated rails; the fused-draft - decision enforces that by declining fusion otherwise, and a private-pool - draft indexes its own pool by virtual id.""" + Only the target's prefill/decode pair is checked here: fusion declines for + a draft off the translated rails, and a private-pool draft indexes its own + pool by virtual id.""" if allowed is None: allowed = _SPEC_VERIFY_AUDITED_BACKENDS dcp_note = "" @@ -591,8 +590,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: "DFLASH", ), ( "--enable-unified-memory only supports --speculative-algorithm " - "DSPARK (chain draft), DFLASH (fused block draft), and " - "EAGLE/EAGLE3 (fused draft KV); other speculative algorithms are " + "DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are " "not yet audited for the unified pool's virtual-to-physical loc " "translation. Got " f"--speculative-algorithm={cfg.speculative_algorithm!r}." @@ -627,9 +625,7 @@ def handle_unified_memory_pool(server_args: Any) -> None: f"spec-verify-audited attention backends {sorted(eagle_allowed)} " f"(got {sorted(eagle_backends, key=str)})." ) - # An unset draft backend inherits the target's pair. A fused draft is - # MHA-shaped on every host, and these rails also serve a draft that - # keeps an MLA pool of its own. + # An unset draft backend inherits the target's pair. draft_backend = cfg.speculative_draft_attention_backend draft_backends = {draft_backend} if draft_backend else eagle_backends assert draft_backends <= TRANSLATED_MHA_RAILS, ( @@ -649,9 +645,8 @@ def handle_unified_memory_pool(server_args: Any) -> None: ) assert not cfg.enable_two_batch_overlap, ( "--enable-unified-memory does not support --enable-two-batch-overlap: " - "TBO's replay split hands each child a view without the pre-translate " - "write loc, so a captured decode replay raises. " - "TODO(ch-wan): carry out_cache_loc_virtual into the child view." + "the iteration's read tables are indexed by the whole batch's rows, and " + "a TBO child's readers do not offset into them." ) assert not cfg.enable_lmcache, ( "--enable-unified-memory is not yet compatible with --enable-lmcache: " @@ -679,8 +674,8 @@ def handle_unified_memory_pool(server_args: Any) -> None: ), ( "--enable-unified-memory + EAGLE/EAGLE3 does not support " "--disaggregation-decode-retraction-backup=host_pool: the backup builds " - "the draft's host pool off a device pool of its own, and an EAGLE draft " - "on the unified pool lives fused in the target's pages, without one." + "the draft's host pool off the draft's own device pool, which a fused " + "EAGLE draft does not have." ) assert not ( cfg.speculative_algorithm in ("EAGLE", "EAGLE3") diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index a6851c6c15..5086ea8063 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -698,6 +698,14 @@ class TboForwardBatchPreparer: start_token_index:end_token_index ] output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical + # Unified memory refuses two-batch overlap, so the plan's reads stay in + # `req_to_token`, which each child indexes at its own rows; each child + # writes its own tokens of the plan's window. + output_dict["kv_loc_plan"] = batch.kv_loc_plan + if batch.kv_loc_plan is not None: + output_dict["kv_loc_cols"] = batch.kv_loc_plan.cols_slice( + batch.kv_loc_cols, slice(start_token_index, end_token_index) + ) attention_tp_size = get_parallel().attn_tp_size _tbo_padded_len = ( diff --git a/python/sglang/srt/dllm/attention.py b/python/sglang/srt/dllm/attention.py index ad8e77b57e..3eb60a733e 100644 --- a/python/sglang/srt/dllm/attention.py +++ b/python/sglang/srt/dllm/attention.py @@ -18,6 +18,7 @@ from sglang.srt.layers.attention.triton_backend import ( ForwardMetadata, update_sliding_window_buffer, ) +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.runner_utils.capture_mode import ( get_capture_attention_variant, @@ -202,12 +203,10 @@ class DllmFlashAttention: swa_out_cache_loc=torch.empty_like(batch.out_cache_loc) if backend.use_sliding_window_kv_pool else None, - out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc) - if backend.kv_index_translator.is_translating - else None, ) metadata = self._prefill_graph_metadata[key] backend._fill_kv_indptr_and_indices( + batch.kv_loc_plan, bs, batch.extend_prefix_lens, batch.req_pool_indices, @@ -220,6 +219,7 @@ class DllmFlashAttention: _, _, _, offsets = update_sliding_window_buffer( metadata.window_kv_indptr, backend.kv_index_translator, + batch.kv_loc_plan, batch.req_pool_indices, backend.sliding_window_size, batch.extend_prefix_lens, @@ -231,15 +231,7 @@ class DllmFlashAttention: metadata.window_kv_offsets.copy_(offsets) if metadata.swa_out_cache_loc is not None: metadata.swa_out_cache_loc.copy_( - backend.kv_index_translator.sliding_window_write_loc_for( - batch.out_cache_loc - ) - ) - if metadata.out_cache_loc_full_physical is not None: - backend.kv_index_translator.fill_capture_write_loc( - out=metadata.out_cache_loc_full_physical, - forward_batch=batch, - width=metadata.out_cache_loc_full_physical.numel(), + backend.kv_index_translator.write_ids(batch, IdSpaceKind.SLIDING_WINDOW) ) backend.forward_metadata = metadata diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 8080934139..9fb035a114 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -40,6 +40,7 @@ from sglang.srt.utils import is_gfx95_supported if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.speculative.spec_info import SpecInput @@ -1951,6 +1952,7 @@ class AiterAttnBackend(AttentionBackend): paged_kernel_lens=forward_batch.seq_lens, paged_kernel_lens_sum=forward_batch.seq_lens_sum, translator=self.kv_index_translator, + plan=forward_batch.kv_loc_plan, ) ) self.forward_metadata = ForwardMetadata( @@ -2150,6 +2152,7 @@ class AiterAttnBackend(AttentionBackend): max(forward_batch.extend_seq_lens_cpu), forward_batch.seq_lens_cpu.max().item(), spec_info=None, + plan=forward_batch.kv_loc_plan, ) max_q_len = self.mla_indices_updater_prefill.max_q_len @@ -2190,6 +2193,7 @@ class AiterAttnBackend(AttentionBackend): prefix_lens, encoder_lens=forward_batch.encoder_lens, spec_info=None, + plan=forward_batch.kv_loc_plan, ) if self.use_sliding_window_kv_pool: @@ -4454,6 +4458,8 @@ class AiterIndicesUpdaterPrefill: prefix_lens: torch.Tensor, encoder_lens: Optional[torch.Tensor], spec_info: Optional[SpecInput], + *, + plan: KVLocPlan, ): # Keep the signature for type checking. It will be assigned during runtime. raise NotImplementedError() @@ -4466,6 +4472,8 @@ class AiterIndicesUpdaterPrefill: prefix_lens: torch.Tensor, encoder_lens: Optional[torch.Tensor], spec_info: Optional[SpecInput], + *, + plan: KVLocPlan, ): kv_start_idx = None kv_indptr = self.kv_indptr @@ -4515,6 +4523,7 @@ class AiterIndicesUpdaterPrefill: paged_kernel_lens=paged_kernel_lens, paged_kernel_lens_sum=paged_kernel_lens_sum, translator=self.kv_index_translator, + plan=plan, ) ) @@ -4547,6 +4556,8 @@ class AiterMlaIndicesUpdaterPrefill: max_q_len: int, max_kv_len: int, spec_info: Optional[SpecInput], + *, + plan: KVLocPlan, ): # Keep the signature for type checking. It will be assigned during runtime. raise NotImplementedError() @@ -4560,6 +4571,8 @@ class AiterMlaIndicesUpdaterPrefill: max_q_len: int, max_kv_len: int, spec_info: Optional[SpecInput], + *, + plan: KVLocPlan, ): bs = len(req_pool_indices) @@ -4594,6 +4607,7 @@ class AiterMlaIndicesUpdaterPrefill: paged_kernel_lens=kv_lens, paged_kernel_lens_sum=kv_lens_sum, translator=self.kv_index_translator, + plan=plan, ) ) @@ -4663,18 +4677,21 @@ class AiterMultiStepDraftBackend: if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT else 1 ) - v2p = self.kv_index_translator.full_flat_v2p() + src = self.kv_index_translator.read_source( + forward_batch.kv_loc_plan, + req_pool_indices=forward_batch.req_pool_indices, + bs=num_seqs, + ) self.generate_draft_decode_kv_indices[ (self.speculative_num_steps * num_token_blocks, num_seqs, self.topk) ]( - forward_batch.req_pool_indices, - self.req_to_token_pool.req_to_token, + src.row_ids, + src.ids, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, forward_batch.positions, - v2p, - self.pool_len, + src.row_stride, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], triton.next_power_of_2(num_seqs), @@ -4684,7 +4701,9 @@ class AiterMultiStepDraftBackend: # A single token block is the historical launch; NUM_STEPS=0 keeps # its 128-wide program instead of the token-block specialization. NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0, - TRANSLATE=v2p is not None, + ENTRY_PAGE_SIZE=src.entry_page_size, + v2p=src.v2p, + TRANSLATE=src.v2p is not None, ) for i in range(self.speculative_num_steps - 1): diff --git a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py index 22582e7da4..ea867c7c05 100644 --- a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py +++ b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py @@ -34,6 +34,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import ( ) from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run from sglang.srt.mem_cache.layout.paged_view import paged_row_view +from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_flashinfer_available @@ -176,7 +177,7 @@ class CuteDslMLABackend(TRTLLMMLABackend): assert q_rope is not None and k_rope is not None if cos_sin_cache is None: if save_kv_cache and self._fused_set_kv_concat_q_fp8: - loc = self._resolve_fused_write_loc(forward_batch) + loc = forward_batch.out_cache_loc if loc is not None: query = self._set_kv_and_concat_q_fp8_fused( layer=layer, @@ -208,7 +209,7 @@ class CuteDslMLABackend(TRTLLMMLABackend): assert k is not None and k_rope is not None self.token_to_kv_pool.set_mla_kv_buffer( layer, - self._kv_write_loc(forward_batch), + KVWriteLoc.for_batch(forward_batch), k, k_rope, ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 05ba0aca40..6bdddab6ba 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -33,7 +33,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.kv_index_translator import KVReadTables +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -54,6 +54,7 @@ from sglang.srt.utils import get_compiler_backend from sglang.srt.utils.common import get_device_capability if TYPE_CHECKING: + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin from sglang.srt.model_executor.model_runner import ModelRunner @@ -466,10 +467,9 @@ class FlashAttentionBackend(AttentionBackend): if not forward_batch.forward_mode.is_draft_extend_v2(): return if self.kv_index_translator.is_translating: - # The draft-extend runners record this call unconditionally, but a - # translating backend rebuilds these tables out of graph on every - # replay; a captured raw req_to_token gather would then overwrite - # them with virtual ids. + # The draft-extend runners record this call unconditionally; a + # translating backend rebuilds these tables out of graph, and a + # captured req_to_token gather would overwrite them with virtual ids. return bs = forward_batch.batch_size metadata = self.draft_extend_metadata[bs] @@ -543,7 +543,11 @@ class FlashAttentionBackend(AttentionBackend): self._init_full_cg_decode_metadata(forward_batch, in_capture) def _get_swa_write_locations(self, forward_batch: ForwardBatch): - locations = getattr(forward_batch, "out_cache_loc", None) + """This step's sliding-window write ids: the plan's, sliced per + draft step the way `out_cache_loc` is.""" + locations = self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW + ) if ( locations is not None and forward_batch.forward_mode.is_decode_or_idle() @@ -586,11 +590,7 @@ class FlashAttentionBackend(AttentionBackend): if in_capture: self.cuda_graph_swa_out_cache_loc[:n].zero_() else: - self.cuda_graph_swa_out_cache_loc[:n].copy_( - self.kv_index_translator.sliding_window_write_loc_for( - swa_out_cache_loc - ) - ) + self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc) if in_capture: num_tokens = forward_batch.positions.numel() @@ -627,6 +627,7 @@ class FlashAttentionBackend(AttentionBackend): return self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, @@ -678,6 +679,7 @@ class FlashAttentionBackend(AttentionBackend): # A stale non-None seq_lens_cpu buffer would under-size max_seq_pages # (stale page-table rows -> OOB); force None under sync-free. self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, @@ -755,22 +757,17 @@ class FlashAttentionBackend(AttentionBackend): translating = self.kv_index_translator.is_translating max_seq_len_k = int(forward_batch.seq_lens_cpu[:bs].max().item()) if translating: - # Unified pool: the block table is a TRANSLATED page table, built - # straight into these capture-stable buffers from the LIVE v2p, so - # a page relocated by compaction since capture is picked up. Same - # substitution the eager extend branch makes in its `_unified_read` - # fixup; `build_index_table` emits page-granular physical ids - # directly, so there is no `// page_size` to undo. - self.kv_index_translator.build_index_table( - req_pool_indices=forward_batch.req_pool_indices[:bs], - seq_lens=forward_batch.seq_lens[:bs], - into=KVReadTables( - full=m.page_table, - sliding_window=( - m.swa_page_table if self.use_sliding_window_kv_pool else None - ), - ), + # Unified pool: the plan's page table, physical page ids from this + # iteration's v2p, copied into these capture-stable buffers. + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, out=m.page_table ) + if self.use_sliding_window_kv_pool: + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, + out=m.swa_page_table, + kind=IdSpaceKind.SLIDING_WINDOW, + ) elif max_seq_len_k > 0: # Build the block table like the eager extend branch: take every # page_size-th token slot from req_to_token and divide by page_size. @@ -797,22 +794,15 @@ class FlashAttentionBackend(AttentionBackend): self.full_cg_prefill_swa_out_cache_loc.shape[0], "full-CG prefill SWA write-location buffer", ) - # Under the unified pool `out_cache_loc` was rebound to FULL-side - # physical ids at ForwardBatch construction, so the full->swa map - # cannot be re-run on it: a full physical page number is also a - # valid virtual page of the swa v2p, so the map would silently - # resolve it to an unrelated token's swa slot. Phase 2 of the write - # contract derives the swa loc from them instead. - swa_write_loc = ( - self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc + if in_capture: + # The runner-built capture batch writes the page-0 sink. + self.full_cg_prefill_swa_out_cache_loc[:num_out].zero_() + else: + self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_( + self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW + ) ) - if translating - else self.token_to_kv_pool.translate_loc_from_full_to_swa( - forward_batch.out_cache_loc - ) - ) - self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(swa_write_loc) # Captured kernels read the full bucket. Route its inactive tail to # SWA's zero dummy slot to prevent stale writes into live slots. self.full_cg_prefill_swa_out_cache_loc[num_out:].zero_() @@ -826,17 +816,14 @@ class FlashAttentionBackend(AttentionBackend): def _spec_read_seq_len_delta( self, forward_mode: ForwardMode, spec_info: Optional[SpecInput] ) -> int: - """Columns past ``seq_lens`` this mode's whole-sequence read covers — - the translated table's fill must reach ``cache_seqlens``. Takes the - two fields rather than a ForwardBatch: the captured build slices its - batch and has none. Now read by both builds, so eager and replay - widen identically. + """Columns past ``seq_lens`` this mode's whole-sequence read covers; + the translated table must reach ``cache_seqlens``. Zero for the prefix-only shapes: normal decode/extend, the topk>1 - split (drafts read via the expand metadata), draft-extend whose lens - already include the window, and draft-extend's idle batch. The ragged - verify layout's per-row lens are bounded by the draft window, so its - upper bound rides the same delta.""" + split (drafts read via the expand metadata), draft-extend (whose lens + already include the window) and its idle batch. The ragged verify + layout's per-row lens are bounded by the draft window, so the same + delta bounds it.""" if spec_info is None: return 0 if forward_mode.is_target_verify() and self.topk <= 1: @@ -1237,27 +1224,20 @@ class FlashAttentionBackend(AttentionBackend): else None ) if _unified_read: - delta = self._spec_read_seq_len_delta( - forward_batch.forward_mode, forward_batch.spec_info + # The plan's table covers this forward's reads, a verify's and a + # draft decode's window included; the kernels bound each row by + # `cache_seqlens`. + kv_view = self.kv_index_translator.read_table( + forward_batch.kv_loc_plan, rows=forward_batch.batch_size ) - if delta: - kv_view = self.kv_index_translator.widened_index_table( - forward_batch, seq_len_delta=delta - ) - else: - kv_view = self.kv_index_translator.index_table_for_batch(forward_batch) metadata.page_table = kv_view.ids if self.use_sliding_window_kv_pool: - metadata.swa_page_table = kv_view.sliding_window_ids - if swa_out_cache_loc is not None: - # The swa write loc was computed from the still-VIRTUAL - # loc at ForwardBatch construction; re-running the - # full->swa map on the physical loc would be garbage. - metadata.swa_out_cache_loc = ( - self.kv_index_translator.sliding_window_write_loc_for( - swa_out_cache_loc - ) - ) + metadata.swa_page_table = self.kv_index_translator.read_table( + forward_batch.kv_loc_plan, + kind=IdSpaceKind.SLIDING_WINDOW, + rows=forward_batch.batch_size, + ).ids + metadata.swa_out_cache_loc = swa_out_cache_loc elif self.use_sliding_window_kv_pool: # FA3 requires an int32 page_table. metadata.swa_page_table = ( @@ -1265,12 +1245,7 @@ class FlashAttentionBackend(AttentionBackend): metadata.page_table ).to(torch.int32) ) - if swa_out_cache_loc is not None: - metadata.swa_out_cache_loc = ( - self.token_to_kv_pool.translate_loc_from_full_to_swa( - swa_out_cache_loc - ) - ) + metadata.swa_out_cache_loc = swa_out_cache_loc # Logical-page KV sharding: capture the batch's gather plan and swap the # page table to scratch rows. During a sharded extend, attention reads @@ -2864,6 +2839,7 @@ class FlashAttentionBackend(AttentionBackend): def _set_decode_page_metadata( self, + plan: KVLocPlan, metadata, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -2871,9 +2847,8 @@ class FlashAttentionBackend(AttentionBackend): ) -> None: """Fill `cache_seqlens_int32`, `cu_seqlens_k` and the page table(s). - Under the unified pool the translator writes the page tables in place, - so the fused kernel is left with the prefix sum alone -- one pass over - the rows instead of a translated build plus a verbatim copy of it. + Under the unified pool the page tables are the plan's, copied in, so + the fused kernel is left with the prefix sum alone. """ translated = self.kv_index_translator.reads_are_translated normal_decode_set_metadata( @@ -2891,17 +2866,20 @@ class FlashAttentionBackend(AttentionBackend): skip_page_table=translated, ) if translated: - # Fill to `cache_seqlens_int32`, which the kernels bound their reads - # by: a draft decode reads `seq_len_delta` past `seq_lens`. - self.kv_index_translator.fill_read_table( - out=metadata.page_table, - sliding_window_out=metadata.swa_page_table, - req_pool_indices=req_pool_indices, - seq_lens=metadata.cache_seqlens_int32, - ) + # The plan's table reaches `cache_seqlens_int32`, which the kernels + # bound their reads by: a draft decode reads past `seq_lens`. + bs = int(req_pool_indices.numel()) + self.kv_index_translator.copy_page_table(plan, out=metadata.page_table[:bs]) + if metadata.swa_page_table is not None: + self.kv_index_translator.copy_page_table( + plan, + out=metadata.swa_page_table[:bs], + kind=IdSpaceKind.SLIDING_WINDOW, + ) def _apply_cuda_graph_metadata( self, + plan: KVLocPlan, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -2937,6 +2915,7 @@ class FlashAttentionBackend(AttentionBackend): # max_seq_len_k left unset -- unread here (scheduler_metadata # is normal-decode-only). self._set_decode_page_metadata( + plan, metadata, req_pool_indices, seq_lens, @@ -3041,6 +3020,7 @@ class FlashAttentionBackend(AttentionBackend): else self.max_context_len ) self._set_decode_page_metadata( + plan, metadata, req_pool_indices, seq_lens, @@ -3092,7 +3072,7 @@ class FlashAttentionBackend(AttentionBackend): # max_seq_len_k left unset -- unread here (scheduler_metadata is # normal-decode-only). self._set_decode_page_metadata( - metadata, req_pool_indices, verify_lens, verify_delta + plan, metadata, req_pool_indices, verify_lens, verify_delta ) else: # When topk > 1, we need two specific target verify metadata, and then merge states @@ -3225,23 +3205,19 @@ class FlashAttentionBackend(AttentionBackend): metadata.max_seq_len_k + self.page_size - 1 ) // self.page_size if self.kv_index_translator.reads_are_translated: - # The translator writes kernel-facing page ids straight into - # the captured tables; `sliding_window_out` fills the swa twin - # in the same pass. - self.kv_index_translator.fill_read_table( - out=metadata.page_table, - sliding_window_out=( - metadata.swa_page_table - if self.use_sliding_window_kv_pool - and metadata.swa_page_table is not None - else None - ), - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - seq_len_delta=self._spec_read_seq_len_delta( - forward_mode, spec_info - ), + rows = int(req_pool_indices.numel()) + self.kv_index_translator.copy_page_table( + plan, out=metadata.page_table[:rows] ) + if ( + self.use_sliding_window_kv_pool + and metadata.swa_page_table is not None + ): + self.kv_index_translator.copy_page_table( + plan, + out=metadata.swa_page_table[:rows], + kind=IdSpaceKind.SLIDING_WINDOW, + ) else: page_indices = self.req_to_token[ req_pool_indices[:, None], diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 5fb4ca1ae7..64615260c8 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -35,6 +35,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( ) from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.model_executor.cuda_graph_config import ( Backend, @@ -61,6 +62,7 @@ from sglang.srt.utils import ( if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner logger = logging.getLogger(__name__) @@ -749,6 +751,7 @@ class FlashInferAttnBackend(AttentionBackend): fixed_split_size=None, disable_split_kv=self.disable_cuda_graph_kv_split, req_pool_indices=req_pool_indices, + plan=forward_batch.kv_loc_plan, ) elif forward_mode.is_target_verify(): self.indices_updater_prefill.update( @@ -761,6 +764,7 @@ class FlashInferAttnBackend(AttentionBackend): use_ragged=False, encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, spec_info=spec_info, + plan=forward_batch.kv_loc_plan, ) elif forward_mode.is_dllm_extend(): self.indices_updater_prefill.update( @@ -773,6 +777,7 @@ class FlashInferAttnBackend(AttentionBackend): use_ragged=not self.use_paged, encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, spec_info=None, + plan=forward_batch.kv_loc_plan, ) elif forward_mode.is_draft_extend_v2(): self.indices_updater_prefill.update( @@ -785,6 +790,7 @@ class FlashInferAttnBackend(AttentionBackend): use_ragged=False, encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, spec_info=spec_info, + plan=forward_batch.kv_loc_plan, ) elif forward_mode.is_extend(): # Plain EXTEND under full prefill CUDA graph. plan() runs @@ -803,6 +809,7 @@ class FlashInferAttnBackend(AttentionBackend): use_ragged=False, encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, spec_info=None, + plan=forward_batch.kv_loc_plan, ) else: raise ValueError("Invalid forward mode") @@ -863,8 +870,8 @@ class FlashInferAttnBackend(AttentionBackend): self.cuda_graph_swa_out_cache_loc[:n].zero_() else: self.cuda_graph_swa_out_cache_loc[:n].copy_( - self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc + self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) ) if in_capture: @@ -955,8 +962,8 @@ class FlashInferAttnBackend(AttentionBackend): def init_forward_metadata(self, forward_batch: ForwardBatch): swa_out_cache_loc = None if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None: - swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc + swa_out_cache_loc = self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) if forward_batch.forward_mode.is_decode_or_idle(): @@ -970,6 +977,7 @@ class FlashInferAttnBackend(AttentionBackend): fixed_split_size=self.decode_split_tile_size, disable_split_kv=False, req_pool_indices=forward_batch.req_pool_indices, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = DecodeMetadata( self.decode_wrappers, swa_out_cache_loc=swa_out_cache_loc @@ -985,6 +993,7 @@ class FlashInferAttnBackend(AttentionBackend): use_ragged=False, encoder_lens=forward_batch.encoder_lens, spec_info=forward_batch.spec_info, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = PrefillMetadata( self.prefill_wrappers_verify, @@ -1038,6 +1047,7 @@ class FlashInferAttnBackend(AttentionBackend): cross_attention_custom_mask=forward_batch.cross_attention_custom_mask, extend_prefix_lens_cpu=forward_batch.extend_prefix_lens_cpu, custom_kv_indices=self.dq_page_table, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = PrefillMetadata( self.prefill_wrappers_paged, @@ -1581,6 +1591,7 @@ class FlashInferIndicesUpdaterDecode: fixed_split_size: Optional[int] = None, disable_split_kv: Optional[bool] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, ): # Keep the signature for type checking. It will be assigned during runtime. @@ -1597,6 +1608,7 @@ class FlashInferIndicesUpdaterDecode: fixed_split_size: Optional[int] = None, disable_split_kv: Optional[bool] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, ): decode_wrappers = decode_wrappers or self.decode_wrappers @@ -1610,6 +1622,7 @@ class FlashInferIndicesUpdaterDecode: seq_lens_cpu, fixed_split_size=fixed_split_size, disable_split_kv=disable_split_kv, + plan=plan, req_pool_indices=req_pool_indices, ) @@ -1624,6 +1637,7 @@ class FlashInferIndicesUpdaterDecode: fixed_split_size: Optional[int] = None, disable_split_kv: Optional[bool] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, ): assert self.sliding_window_size is not None @@ -1663,6 +1677,7 @@ class FlashInferIndicesUpdaterDecode: use_sliding_window_kv_pool=use_sliding_window_kv_pool, fixed_split_size=fixed_split_size, disable_split_kv=disable_split_kv, + plan=plan, req_pool_indices=req_pool_indices, ) @@ -1677,6 +1692,7 @@ class FlashInferIndicesUpdaterDecode: fixed_split_size: Optional[int] = None, disable_split_kv: Optional[bool] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, ): # Cache encoder_lens on CPU to avoid GPU→CPU transfer per call @@ -1703,6 +1719,7 @@ class FlashInferIndicesUpdaterDecode: seq_lens_cpu=kv_lens_cpu, fixed_split_size=fixed_split_size, disable_split_kv=disable_split_kv, + plan=plan, req_pool_indices=req_pool_indices, ) @@ -1719,6 +1736,7 @@ class FlashInferIndicesUpdaterDecode: fixed_split_size: Optional[int] = None, disable_split_kv: Optional[bool] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, ): # Unified SWA wrapper-0: gather from the swa canonical directly -- its @@ -1726,6 +1744,7 @@ class FlashInferIndicesUpdaterDecode: # full->swa translate below must not run on top of them. translator = self.attn_backend.kv_index_translator use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated + read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL if spec_info is None or getattr(spec_info, "kv_indptr", None) is None: bs = len(paged_kernel_lens) kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) @@ -1739,14 +1758,14 @@ class FlashInferIndicesUpdaterDecode: paged_kernel_lens_sum, dtype=torch.int32, device="cuda" ) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=kv_indptr, - total_tokens=paged_kernel_lens_sum, out=kv_indices, kv_start_idx=kv_start_idx, - sliding_window=use_swa_source, + kind=read_kind, ) else: kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices @@ -1868,6 +1887,8 @@ class FlashInferIndicesUpdaterPrefill: cross_attention_custom_mask: Optional[torch.Tensor] = None, extend_prefix_lens_cpu: Optional[List[int]] = None, custom_kv_indices: Optional[torch.Tensor] = None, + *, + plan: KVLocPlan, ): # Keep the signature for type checking. It will be assigned during runtime. raise NotImplementedError() @@ -1888,6 +1909,8 @@ class FlashInferIndicesUpdaterPrefill: cross_attention_custom_mask: Optional[torch.Tensor] = None, extend_prefix_lens_cpu: Optional[List[int]] = None, custom_kv_indices: Optional[torch.Tensor] = None, + *, + plan: KVLocPlan, ): if use_ragged: assert prefix_lens is not None @@ -1918,6 +1941,7 @@ class FlashInferIndicesUpdaterPrefill: multi_item_params=multi_item_params, seq_lens_cpu=seq_lens_cpu, custom_kv_indices=custom_kv_indices, + plan=plan, ) def update_sliding_window( @@ -1936,6 +1960,8 @@ class FlashInferIndicesUpdaterPrefill: cross_attention_custom_mask: Optional[torch.Tensor] = None, extend_prefix_lens_cpu: Optional[List[int]] = None, custom_kv_indices: Optional[torch.Tensor] = None, + *, + plan: KVLocPlan, ): if custom_kv_indices is not None: raise RuntimeError( @@ -2031,6 +2057,7 @@ class FlashInferIndicesUpdaterPrefill: if (wrapper_id == 0 and not use_ragged and spec_info is None) else -1 ), + plan=plan, ) def _build_swa_prefix_custom_mask( @@ -2090,6 +2117,8 @@ class FlashInferIndicesUpdaterPrefill: cross_attention_custom_mask: Optional[torch.Tensor] = None, extend_prefix_lens_cpu: Optional[List[int]] = None, custom_kv_indices: Optional[torch.Tensor] = None, + *, + plan: KVLocPlan, ): if custom_kv_indices is not None: raise RuntimeError( @@ -2125,6 +2154,7 @@ class FlashInferIndicesUpdaterPrefill: cross_attention_custom_mask=( cross_attention_custom_mask if wrapper_id == 1 else None ), + plan=plan, ) def call_begin_forward( @@ -2148,6 +2178,8 @@ class FlashInferIndicesUpdaterPrefill: seq_lens_cpu: Optional[torch.Tensor] = None, custom_kv_indices: Optional[torch.Tensor] = None, window_left: int = -1, + *, + plan: KVLocPlan, ): bs = len(seq_lens) # Unified SWA wrapper-0: gather from the swa canonical directly -- its @@ -2155,6 +2187,7 @@ class FlashInferIndicesUpdaterPrefill: # full->swa translate below must not run on top of them. translator = self.attn_backend.kv_index_translator use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated + read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL if spec_info is None: assert prefix_lens is not None assert len(seq_lens) == len(req_pool_indices) @@ -2180,14 +2213,14 @@ class FlashInferIndicesUpdaterPrefill: dtype=torch.int32, device=req_pool_indices.device, ) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=kv_indptr, - total_tokens=paged_kernel_lens_sum, out=kv_indices, kv_start_idx=kv_start_idx, - sliding_window=use_swa_source, + kind=read_kind, ) qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0) qo_indptr = qo_indptr[: bs + 1] @@ -2202,7 +2235,8 @@ class FlashInferIndicesUpdaterPrefill: paged_kernel_lens=paged_kernel_lens, paged_kernel_lens_sum=paged_kernel_lens_sum, translator=translator, - sliding_window=use_swa_source, + plan=plan, + kind=read_kind, kv_start_idx=kv_start_idx, ) ) @@ -2213,7 +2247,8 @@ class FlashInferIndicesUpdaterPrefill: paged_kernel_lens=paged_kernel_lens, paged_kernel_lens_sum=paged_kernel_lens_sum, translator=translator, - sliding_window=use_swa_source, + plan=plan, + kind=read_kind, ) ) @@ -2392,18 +2427,21 @@ class FlashInferMultiStepDraftBackend: seq_lens_sum=seq_lens_sum, ) - v2p = self.kv_index_translator.full_flat_v2p() + src = self.kv_index_translator.read_source( + forward_batch.kv_loc_plan, + req_pool_indices=forward_batch.req_pool_indices, + bs=num_seqs, + ) self.generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( - forward_batch.req_pool_indices, - self.req_to_token_pool.req_to_token, + src.row_ids, + src.ids, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, forward_batch.positions, - v2p, - self.pool_len, + src.row_stride, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], next_power_of_2(num_seqs), @@ -2412,7 +2450,9 @@ class FlashInferMultiStepDraftBackend: self.page_size, self.draft_window_size, self.draft_sink_size, - TRANSLATE=v2p is not None, + ENTRY_PAGE_SIZE=src.entry_page_size, + v2p=src.v2p, + TRANSLATE=src.v2p is not None, ) assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 6a68be22f2..6944310f68 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -57,6 +57,7 @@ if TYPE_CHECKING: FlashInferMlaAttnBackend, ) from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.speculative.spec_info import SpecInput @@ -367,6 +368,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): init_metadata_replay=False, spec_info=spec_info, req_pool_indices=req_pool_indices[:bs], + plan=forward_batch.kv_loc_plan, ) self.decode_cuda_graph_metadata[bs] = decode_wrapper self.forward_metadata = DecodeMetadata(decode_wrapper) @@ -391,6 +393,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): raise ValueError(f"Invalid mode: {forward_mode=}") self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, @@ -408,6 +411,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): prefill_wrapper.plan = partial(fast_mla_prefill_plan, prefill_wrapper) else: self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, @@ -425,6 +429,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): decode_wrapper=self.decode_wrapper, init_metadata_replay=False, req_pool_indices=forward_batch.req_pool_indices, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = DecodeMetadata(self.decode_wrapper) elif forward_batch.forward_mode.is_target_verify(): @@ -436,6 +441,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): prefill_wrapper_paged=self.prefill_wrapper_verify, use_ragged=False, spec_info=forward_batch.spec_info, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False) else: @@ -484,6 +490,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): qo_indptr_cpu=qo_indptr_cpu, kv_indptr_cpu=kv_indptr_cpu, kv_len_arr_cpu=kv_len_arr_cpu, + plan=forward_batch.kv_loc_plan, ) self.forward_metadata = PrefillMetadata( self.prefill_wrapper_paged, use_ragged @@ -531,6 +538,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): def _apply_cuda_graph_metadata( self, + plan: KVLocPlan, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -566,6 +574,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): init_metadata_replay=True, spec_info=spec_info, req_pool_indices=req_pool_indices[:bs], + plan=plan, **self.fast_decode_kwargs, ) elif forward_mode.is_target_verify(): @@ -616,6 +625,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): if use_generic_fast_plan else None ), + plan=plan, ) else: raise ValueError(f"Invalid forward mode: {forward_mode=}") @@ -859,6 +869,7 @@ class FlashInferMLAIndicesUpdaterDecode: init_metadata_replay: bool = False, spec_info: Optional[SpecInput] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, **fast_decode_kwargs, ): @@ -871,6 +882,7 @@ class FlashInferMLAIndicesUpdaterDecode: self.kv_indptr, init_metadata_replay, spec_info, + plan=plan, req_pool_indices=req_pool_indices, **fast_decode_kwargs, ) @@ -885,6 +897,7 @@ class FlashInferMLAIndicesUpdaterDecode: init_metadata_replay: bool = False, spec_info: Optional[SpecInput] = None, *, + plan: KVLocPlan, req_pool_indices: torch.Tensor, **fast_decode_kwargs, ): @@ -904,11 +917,11 @@ class FlashInferMLAIndicesUpdaterDecode: else fast_decode_kwargs["kv_indices"] ) translator = self.attn_backend.kv_index_translator - is_translated = translator.fill_packed_read_stream( + is_translated = translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=kv_indptr, - total_tokens=paged_kernel_lens_sum, out=kv_indices, ) @@ -1006,6 +1019,7 @@ class FlashInferMLAIndicesUpdaterPrefill: attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None, fast_verify_plan_kwargs: Optional[dict] = None, *, + plan: KVLocPlan, qo_indptr_cpu: Optional[torch.Tensor] = None, kv_indptr_cpu: Optional[torch.Tensor] = None, kv_len_arr_cpu: Optional[torch.Tensor] = None, @@ -1031,6 +1045,7 @@ class FlashInferMLAIndicesUpdaterPrefill: spec_info, attn_dcp_metadata=attn_dcp_metadata, fast_verify_plan_kwargs=fast_verify_plan_kwargs, + plan=plan, qo_indptr_cpu=qo_indptr_cpu, kv_indptr_cpu=kv_indptr_cpu, kv_len_arr_cpu=kv_len_arr_cpu, @@ -1052,6 +1067,7 @@ class FlashInferMLAIndicesUpdaterPrefill: attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None, fast_verify_plan_kwargs: Optional[dict] = None, *, + plan: KVLocPlan, qo_indptr_cpu: Optional[torch.Tensor] = None, kv_indptr_cpu: Optional[torch.Tensor] = None, kv_len_arr_cpu: Optional[torch.Tensor] = None, @@ -1068,11 +1084,11 @@ class FlashInferMLAIndicesUpdaterPrefill: dtype=torch.int32, device=req_pool_indices.device, ) - self.attn_backend.kv_index_translator.fill_packed_read_stream( + self.attn_backend.kv_index_translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=kv_indptr, - total_tokens=paged_kernel_lens_sum, out=kv_indices, ) qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0) @@ -1085,6 +1101,7 @@ class FlashInferMLAIndicesUpdaterPrefill: paged_kernel_lens=paged_kernel_lens, paged_kernel_lens_sum=paged_kernel_lens_sum, translator=self.attn_backend.kv_index_translator, + plan=plan, kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"], ) ) @@ -1097,6 +1114,7 @@ class FlashInferMLAIndicesUpdaterPrefill: paged_kernel_lens=paged_kernel_lens, paged_kernel_lens_sum=paged_kernel_lens_sum, translator=self.attn_backend.kv_index_translator, + plan=plan, ) ) @@ -1241,25 +1259,30 @@ class FlashInferMLAMultiStepDraftBackend: seq_lens_sum=seq_lens_sum, ) - v2p = self.kv_index_translator.full_flat_v2p() + src = self.kv_index_translator.read_source( + forward_batch.kv_loc_plan, + req_pool_indices=forward_batch.req_pool_indices, + bs=num_seqs, + ) self.generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( - forward_batch.req_pool_indices, - self.req_to_token_pool.req_to_token, + src.row_ids, + src.ids, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, forward_batch.positions, - v2p, - self.pool_len, + src.row_stride, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], next_power_of_2(num_seqs), next_power_of_2(self.speculative_num_steps), next_power_of_2(bs), self.page_size, - TRANSLATE=v2p is not None, + ENTRY_PAGE_SIZE=src.entry_page_size, + v2p=src.v2p, + TRANSLATE=src.v2p is not None, ) assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 57a7bc0b02..daa7c83e32 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -28,6 +28,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner logger = logging.getLogger(__name__) @@ -125,6 +126,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): or forward_mode.is_draft_extend_v2() ): self._apply_decode_target_verify_metadata( + plan=forward_batch.kv_loc_plan, bs=forward_batch.batch_size, req_pool_indices=forward_batch.req_pool_indices, seq_lens=forward_batch.seq_lens, @@ -165,10 +167,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend): block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE - self.kv_index_translator.fill_read_table( - out=block_kv_indices, - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=forward_batch.seq_lens, + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, out=block_kv_indices ) else: create_flashmla_kv_indices_triton[ @@ -200,10 +200,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend): block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE - self.kv_index_translator.fill_read_table( - out=block_kv_indices, - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=seq_lens, + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, out=block_kv_indices ) else: create_flashmla_kv_indices_triton[ @@ -240,10 +238,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend): block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE - self.kv_index_translator.fill_read_table( - out=block_kv_indices, - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=seq_lens_k, + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, out=block_kv_indices ) else: create_flashmla_kv_indices_triton[ @@ -328,6 +324,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): def _apply_decode_target_verify_metadata( self, + plan: KVLocPlan, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -355,10 +352,9 @@ class FlashMLABackend(FlashInferMLAAttnBackend): if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE - self.kv_index_translator.fill_read_table( - out=self.cuda_graph_kv_indices, - req_pool_indices=req_pool_indices[:bs], - seq_lens=seq_lens, + # The plan's table reaches a verify's draft tail. + self.kv_index_translator.copy_page_table( + plan, out=self.cuda_graph_kv_indices[:bs] ) else: create_flashmla_kv_indices_triton[ diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py index e8e83fdf0f..5c6ac3dac4 100644 --- a/python/sglang/srt/layers/attention/tbo_backend.py +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -251,6 +251,7 @@ def _build_tbo_child_replay_fb_view( child_seq_lens_cpu = fb_view.seq_lens_cpu[seq_slice] parent_input_ids = getattr(fb_view, "input_ids", None) parent_out_cache_loc = getattr(fb_view, "out_cache_loc", None) + plan = getattr(fb_view, "kv_loc_plan", None) return SimpleNamespace( batch_size=child_bs, forward_mode=fb_view.forward_mode, @@ -270,6 +271,15 @@ def _build_tbo_child_replay_fb_view( if parent_out_cache_loc is not None else None ), + # Unified memory refuses two-batch overlap, so the plan's reads stay in + # `req_to_token`, which each child indexes at its own rows; each child + # writes its own tokens of the plan's window. + kv_loc_plan=plan, + kv_loc_cols=( + None + if plan is None + else plan.cols_slice(getattr(fb_view, "kv_loc_cols", None), tok_slice) + ), spec_info=child_spec_info, max_seq_len_override=fb_view.max_seq_len_override, ) diff --git a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py index 156e542419..4b36a9506b 100644 --- a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py +++ b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py @@ -404,7 +404,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend): if save_kv_cache: self.token_to_kv_pool.set_mla_kv_buffer( layer, - self._kv_write_loc(forward_batch), + KVWriteLoc.for_batch(forward_batch), k, k_rope, ) diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 9c243377a4..a2a02a396d 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -30,6 +30,7 @@ from sglang.srt.layers.dcp import ( ) from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.cuda_graph_config import ( @@ -68,6 +69,7 @@ if _is_cuda: if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.speculative.spec_info import SpecInput @@ -130,9 +132,6 @@ class ForwardMetadata: swa_attn_logits: Optional[torch.Tensor] = None # full->SWA translated out_cache_loc (SWA KV-store write target) swa_out_cache_loc: Optional[torch.Tensor] = None - # PHYSICAL full-attn write target for the unified pool (eager: translated tensor; - # cuda-graph: capture-stable buffer view). None for non-unified pools. - out_cache_loc_full_physical: Optional[torch.Tensor] = None # Lean decode (persistent-grid partial-result buffers) lean_Mp: Optional[torch.Tensor] = None lean_Lp: Optional[torch.Tensor] = None @@ -503,6 +502,7 @@ class TritonAttnBackend(AttentionBackend): def _fill_kv_indptr_and_indices( self, + plan: KVLocPlan, bs: int, seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, @@ -512,17 +512,18 @@ class TritonAttnBackend(AttentionBackend): if kv_indptr is None: kv_indptr = self.kv_indptr[: bs + 1] kv_indptr[1:] = torch.cumsum(seq_lens, dim=0) - self.kv_index_translator.fill_packed_read_stream( - req_pool_indices=req_pool_indices[:bs], + self.kv_index_translator.pack_read_stream( + plan, + req_pool_indices=req_pool_indices, seq_lens=seq_lens[:bs], indptr=kv_indptr, - total_tokens=kv_indices.numel(), out=kv_indices, ) return kv_indptr def _update_decode_kv_buffers( self, + plan: KVLocPlan, bs: int, seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, @@ -551,7 +552,7 @@ class TritonAttnBackend(AttentionBackend): num_kv_splits_lens = dcp_seq_lens.clamp_min(1) else: kv_indptr = self._fill_kv_indptr_and_indices( - bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices + plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices ) num_kv_splits_lens = seq_lens window_kv_indptr = self.window_kv_indptr @@ -560,6 +561,7 @@ class TritonAttnBackend(AttentionBackend): window_kv_indptr, _, window_kv_lens, _ = update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + plan, req_pool_indices, self.sliding_window_size, seq_lens, @@ -577,6 +579,7 @@ class TritonAttnBackend(AttentionBackend): def _update_target_verify_buffers( self, + plan: KVLocPlan, bs: int, seq_lens: torch.Tensor, spec_info, @@ -593,7 +596,7 @@ class TritonAttnBackend(AttentionBackend): device=self.device, ) kv_indptr = self._fill_kv_indptr_and_indices( - bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices + plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices ) window_kv_indptr = self.window_kv_indptr window_kv_indices = None @@ -607,6 +610,7 @@ class TritonAttnBackend(AttentionBackend): update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + plan, req_pool_indices, self.sliding_window_size, seq_lens[:bs], @@ -638,6 +642,7 @@ class TritonAttnBackend(AttentionBackend): def _update_dllm_buffers( self, + plan: KVLocPlan, bs: int, seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, @@ -654,12 +659,13 @@ class TritonAttnBackend(AttentionBackend): device=self.device, ) self._fill_kv_indptr_and_indices( - bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices + plan, bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices ) if self.sliding_window_size is not None and self.sliding_window_size > 0: update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + plan, req_pool_indices, self.sliding_window_size, prefix_lens, @@ -670,6 +676,7 @@ class TritonAttnBackend(AttentionBackend): def _update_draft_extend_buffers( self, + plan: KVLocPlan, bs: int, seq_lens: torch.Tensor, forward_mode: ForwardMode, @@ -705,7 +712,7 @@ class TritonAttnBackend(AttentionBackend): extend_seq_lens = torch.zeros(bs, dtype=torch.int32, device=seq_lens.device) kv_lens = torch.clamp(seq_lens - extend_seq_lens, min=0).to(torch.int32) kv_indptr = self._fill_kv_indptr_and_indices( - bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices + plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices ) return qo_indptr, kv_indptr, num_tokens_per_req @@ -755,15 +762,13 @@ class TritonAttnBackend(AttentionBackend): return self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, forward_mode=forward_mode, spec_info=spec_info, ) - out_cache_loc_full_physical = self._fill_cuda_graph_write_locs( - forward_batch, bs - ) swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc( forward_batch, in_capture=True ) @@ -772,25 +777,25 @@ class TritonAttnBackend(AttentionBackend): forward_mode, spec_info, swa_out_cache_loc, - out_cache_loc_full_physical, ) else: self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=req_pool_indices, seq_lens=seq_lens, forward_mode=forward_mode, spec_info=spec_info, ) - # Metadata view is reused from capture; just refill the buffers. - self._fill_cuda_graph_write_locs(forward_batch, bs) + # Metadata view is reused from capture; the write loc is the + # runner's slot, so only the swa twin needs refilling. self._fill_cuda_graph_swa_out_cache_loc(forward_batch) def _fill_cuda_graph_swa_out_cache_loc( self, forward_batch: ForwardBatch, in_capture: bool = False ) -> Optional[torch.Tensor]: - """Refill the SWA write-target buffer from the batch's derived - sliding-window write loc, returning the [:n] view (None for non-SWA / + """Refill the SWA write-target buffer with the batch's write ids in the + sliding-window sub-pool, returning the [:n] view (None for non-SWA / multi-step draft) so the captured store reads fresh slots on replay. """ if not self.use_sliding_window_kv_pool: @@ -807,24 +812,12 @@ class TritonAttnBackend(AttentionBackend): self.cuda_graph_swa_out_cache_loc[:n].zero_() else: self.cuda_graph_swa_out_cache_loc[:n].copy_( - self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc) + self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW + ) ) return self.cuda_graph_swa_out_cache_loc[:n] - def _fill_cuda_graph_write_locs( - self, forward_batch: ForwardBatch, bs: int - ) -> Optional[torch.Tensor]: - """Runs BEFORE graph.replay(), so it reads the live post-compaction - v2p; no-op for non-unified pools.""" - # The buffer exists only for a translating pool; return before naming it. - if not self.kv_index_translator.is_translating: - return None - return self.kv_index_translator.fill_capture_write_loc( - out=self.cuda_graph_out_cache_loc_full_physical, - forward_batch=forward_batch, - width=self.cuda_graph_out_cache_loc_full_physical.numel(), - ) - def init_forward_metadata(self, forward_batch: ForwardBatch): """Init auxiliary variables for triton attention backend.""" @@ -862,6 +855,7 @@ class TritonAttnBackend(AttentionBackend): seq_lens_sum, dtype=torch.int64, device=self.device ) kv_indptr = self._fill_kv_indptr_and_indices( + forward_batch.kv_loc_plan, bs, forward_batch.seq_lens, forward_batch.req_pool_indices, @@ -875,6 +869,7 @@ class TritonAttnBackend(AttentionBackend): update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + forward_batch.kv_loc_plan, forward_batch.req_pool_indices, self.sliding_window_size, forward_batch.seq_lens, @@ -961,6 +956,7 @@ class TritonAttnBackend(AttentionBackend): seq_lens_sum, dtype=torch.int64, device=self.device ) kv_indptr = self._fill_kv_indptr_and_indices( + forward_batch.kv_loc_plan, bs, forward_batch.seq_lens, forward_batch.req_pool_indices, @@ -977,6 +973,7 @@ class TritonAttnBackend(AttentionBackend): ) = update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + forward_batch.kv_loc_plan, forward_batch.req_pool_indices, self.sliding_window_size, forward_batch.seq_lens, @@ -1016,6 +1013,7 @@ class TritonAttnBackend(AttentionBackend): device=self.device, ) kv_indptr = self._fill_kv_indptr_and_indices( + forward_batch.kv_loc_plan, bs, forward_batch.extend_prefix_lens, forward_batch.req_pool_indices, @@ -1030,6 +1028,7 @@ class TritonAttnBackend(AttentionBackend): ) = update_sliding_window_buffer( self.window_kv_indptr, self.kv_index_translator, + forward_batch.kv_loc_plan, forward_batch.req_pool_indices, self.sliding_window_size, forward_batch.extend_prefix_lens, @@ -1054,8 +1053,8 @@ class TritonAttnBackend(AttentionBackend): swa_out_cache_loc = None if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None: - swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc + swa_out_cache_loc = self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) self.forward_metadata = ForwardMetadata( @@ -1074,11 +1073,6 @@ class TritonAttnBackend(AttentionBackend): window_kv_offsets, swa_attn_logits=swa_attn_logits, swa_out_cache_loc=swa_out_cache_loc, - out_cache_loc_full_physical=( - forward_batch.out_cache_loc - if self.kv_index_translator.is_translating - else None - ), lean_Mp=lean_Mp, lean_Lp=lean_Lp, lean_Op=lean_Op, @@ -1201,22 +1195,12 @@ class TritonAttnBackend(AttentionBackend): device=self.device, ) - if self.kv_index_translator.is_translating: - # Unified pool full-attention write-target buffer, refilled at replay - # (-> KVWriteLoc.full_loc). Capture-stable, mirrors cuda_graph_swa_out_cache_loc. - self.cuda_graph_out_cache_loc_full_physical = torch.zeros( - (max_num_tokens,), - dtype=torch.int64, - device=self.device, - ) - def _build_cuda_graph_forward_metadata( self, bs: int, forward_mode: ForwardMode, spec_info: Optional[SpecInput], swa_out_cache_loc: Optional[torch.Tensor] = None, - out_cache_loc_full_physical: Optional[torch.Tensor] = None, ) -> ForwardMetadata: """Construct ForwardMetadata from the current cuda-graph buffer state. @@ -1247,7 +1231,6 @@ class TritonAttnBackend(AttentionBackend): window_kv_offsets=None, swa_attn_logits=self.cuda_graph_swa_attn_logits, swa_out_cache_loc=swa_out_cache_loc, - out_cache_loc_full_physical=out_cache_loc_full_physical, lean_Mp=self.cuda_graph_lean_Mp, lean_Lp=self.cuda_graph_lean_Lp, lean_Op=self.cuda_graph_lean_Op, @@ -1279,7 +1262,6 @@ class TritonAttnBackend(AttentionBackend): ), window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None, swa_out_cache_loc=swa_out_cache_loc, - out_cache_loc_full_physical=out_cache_loc_full_physical, ) elif forward_mode.is_dllm_extend(): return ForwardMetadata( @@ -1297,7 +1279,6 @@ class TritonAttnBackend(AttentionBackend): window_num_kv_splits=None, window_kv_offsets=None, swa_out_cache_loc=swa_out_cache_loc, - out_cache_loc_full_physical=out_cache_loc_full_physical, ) elif forward_mode.is_draft_extend_v2(): return ForwardMetadata( @@ -1323,13 +1304,13 @@ class TritonAttnBackend(AttentionBackend): window_num_kv_splits=None, window_kv_offsets=None, swa_out_cache_loc=swa_out_cache_loc, - out_cache_loc_full_physical=out_cache_loc_full_physical, ) else: raise ValueError(f"Invalid forward mode: {forward_mode=} for CUDA Graph.") def _apply_cuda_graph_metadata( self, + plan: KVLocPlan, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -1344,7 +1325,7 @@ class TritonAttnBackend(AttentionBackend): if forward_mode.is_decode_or_idle(): assert spec_info is None, "Multi-step cuda graph init is not done here." _, _, window_kv_lens, num_kv_splits_lens = self._update_decode_kv_buffers( - bs, seq_lens, req_pool_indices + plan, bs, seq_lens, req_pool_indices ) self.get_num_kv_splits( self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs] @@ -1356,13 +1337,13 @@ class TritonAttnBackend(AttentionBackend): elif forward_mode.is_target_verify(): bs = len(req_pool_indices) self._update_target_verify_buffers( - bs, seq_lens, spec_info, req_pool_indices + plan, bs, seq_lens, spec_info, req_pool_indices ) elif forward_mode.is_dllm_extend(): - self._update_dllm_buffers(bs, seq_lens, req_pool_indices) + self._update_dllm_buffers(plan, bs, seq_lens, req_pool_indices) elif forward_mode.is_draft_extend_v2(): self._update_draft_extend_buffers( - bs, seq_lens, forward_mode, spec_info, req_pool_indices + plan, bs, seq_lens, forward_mode, spec_info, req_pool_indices ) else: raise ValueError( @@ -1649,7 +1630,6 @@ class TritonAttnBackend(AttentionBackend): loc_info = KVWriteLoc.for_batch( forward_batch, swa_loc=self.forward_metadata.swa_out_cache_loc, - full_loc=self.forward_metadata.out_cache_loc_full_physical, ) if layer.k_scale is None: self._set_kv_buffer(forward_batch, layer, loc_info, k, v) @@ -2123,8 +2103,6 @@ class TritonAttnBackend(AttentionBackend): "window-layer extend before the metadata carried a " "sliding-window write loc" ) - elif self.forward_metadata.out_cache_loc_full_physical is not None: - extend_kv_indices = self.forward_metadata.out_cache_loc_full_physical # Capture batches may not have a spec_info, so use the attention # metadata's resolved uniform verify width when extend lengths are absent. @@ -2234,14 +2212,9 @@ class TritonAttnBackend(AttentionBackend): k.div_(layer.k_scale) self.token_to_kv_pool.set_kv_buffer( layer, - # `full_loc` carries the pre-translated loc under the unified - # pool, refreshed into a capture-stable buffer before replay — - # translating inside set_kv_buffer would be captured and replay - # a stale v2p. None (-> raw loc) for static pools. KVWriteLoc.for_batch( forward_batch, swa_loc=self.forward_metadata.swa_out_cache_loc, - full_loc=self.forward_metadata.out_cache_loc_full_physical, ), k, v, @@ -2253,7 +2226,6 @@ class TritonAttnBackend(AttentionBackend): KVWriteLoc.for_batch( forward_batch, swa_loc=self.forward_metadata.swa_out_cache_loc, - full_loc=self.forward_metadata.out_cache_loc_full_physical, ), k, v, @@ -2468,18 +2440,21 @@ class TritonMultiStepDraftBackend: # over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H. seq_lens_sum = num_seqs * self.max_context_len - v2p = self.kv_index_translator.full_flat_v2p() + src = self.kv_index_translator.read_source( + forward_batch.kv_loc_plan, + req_pool_indices=forward_batch.req_pool_indices, + bs=num_seqs, + ) generate_draft_decode_kv_indices[ (self.speculative_num_steps, num_seqs, self.topk) ]( - forward_batch.req_pool_indices, - self.req_to_token_pool.req_to_token, + src.row_ids, + src.ids, forward_batch.seq_lens, kv_indices_buffer, self.kv_indptr, forward_batch.positions, - v2p, - self.pool_len, + src.row_stride, kv_indices_buffer.shape[1], self.kv_indptr.shape[1], next_power_of_2(num_seqs), @@ -2488,7 +2463,9 @@ class TritonMultiStepDraftBackend: self.page_size, self.draft_window_size, self.draft_sink_size, - TRANSLATE=v2p is not None, + ENTRY_PAGE_SIZE=src.entry_page_size, + v2p=src.v2p, + TRANSLATE=src.v2p is not None, ) if call_fn is None: @@ -2590,6 +2567,7 @@ class TritonMultiStepDraftBackend: def update_sliding_window_buffer( window_kv_indptr, translator, + plan, req_pool_indices, sliding_window_size, seq_lens, @@ -2626,16 +2604,18 @@ def update_sliding_window_buffer( if isinstance(token_to_kv_pool, SWAKVPool) else None ) - translated = translator.fill_packed_read_stream( - req_pool_indices=req_pool_indices[:bs], + translated = translator.pack_read_stream( + plan, + req_pool_indices=req_pool_indices, seq_lens=window_kv_lens, indptr=window_kv_indptr, - total_tokens=window_kv_indices.numel(), out=window_kv_indices, kv_start_idx=window_kv_start_idx, - sliding_window=( - translator.reads_are_translated + kind=( + IdSpaceKind.SLIDING_WINDOW + if translator.reads_are_translated and isinstance(token_to_kv_pool, BaseSWAKVPool) + else IdSpaceKind.FULL ), token_mapping=token_mapping, ) diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index 17fe7b2ccc..0242936f5f 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import ( KVCacheAttentionAccessKind, ) from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.layout.paged_view import paged_kv_view from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool @@ -1088,16 +1089,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): # fused kernel skips its page-table writes so the graph reads the # refreshed content through pointers baked at capture). metadata = self.forward_metadata - # `cache_seqlens_int32` is what the attention kernels bound their - # page-table reads by, and the fused metadata call above wrote it. - # A target verify reads `draft_token_num` further than `seq_lens` - # goes, so filling to `seq_lens` leaves those columns untranslated. - self.kv_index_translator.fill_read_table( - out=metadata.page_table, - req_pool_indices=forward_batch.req_pool_indices[:bs], - seq_lens=metadata.cache_seqlens_int32, - sliding_window_out=metadata.swa_page_table, + # The kernels bound their page-table reads by `cache_seqlens_int32` + # (the fused metadata call above wrote it); the plan's table + # reaches that far, a target verify's draft tail included. + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, out=metadata.page_table[:bs] ) + if metadata.swa_page_table is not None: + self.kv_index_translator.copy_page_table( + forward_batch.kv_loc_plan, + out=metadata.swa_page_table[:bs], + kind=IdSpaceKind.SLIDING_WINDOW, + ) # A capture batch carries no prepared write loc; zeros are the # page-0 sink. if ( @@ -1110,8 +1113,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): self.cuda_graph_swa_out_cache_loc[:n].zero_() else: self.cuda_graph_swa_out_cache_loc[:n].copy_( - self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc + self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) ) @@ -1153,7 +1156,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): metadata, req_pool_indices, metadata.cache_seqlens_int32 ) # The fused in-graph kernel also skips ragged batches, so refill the - # SWA write-target buffer here (out_cache_loc -> SWA locs). + # SWA write-target buffer here, from the plan. if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None: n = forward_batch.out_cache_loc.shape[0] self.cuda_graph_swa_out_cache_loc[n:].zero_() @@ -1161,8 +1164,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): self.cuda_graph_swa_out_cache_loc[:n].zero_() else: self.cuda_graph_swa_out_cache_loc[:n].copy_( - self.token_to_kv_pool.translate_loc_from_full_to_swa( - forward_batch.out_cache_loc + self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) ) @@ -1266,12 +1269,23 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): else: metadata.cu_seqlens_q = metadata.cu_seqlens_k - kv_view = self.kv_index_translator.index_table_for_batch(forward_batch) + kv_view = self.kv_index_translator.read_table( + forward_batch.kv_loc_plan, rows=forward_batch.batch_size + ) if kv_view.is_translated: # No fill kernel: the kernels take the tensor's own width/stride # and bound their reads by cache_seqlens. metadata.page_table = kv_view.ids - metadata.swa_page_table = kv_view.sliding_window_ids + metadata.swa_page_table = ( + self.kv_index_translator.read_table( + forward_batch.kv_loc_plan, + kind=IdSpaceKind.SLIDING_WINDOW, + rows=forward_batch.batch_size, + ).ids + if self.kv_index_translator.space(IdSpaceKind.SLIDING_WINDOW) + is not None + else None + ) else: has_swa = self._swa_kv_pool is not None metadata.page_table = torch.empty( @@ -1300,10 +1314,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend): # int64 scatter index (unlike the int32 read page table above). if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None: - metadata.swa_out_cache_loc = ( - self.kv_index_translator.sliding_window_write_loc_for( - forward_batch.out_cache_loc - ) + metadata.swa_out_cache_loc = self.kv_index_translator.write_ids( + forward_batch, IdSpaceKind.SLIDING_WINDOW ) self.forward_metadata = metadata diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index e3d2f768a7..18ea875fdc 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -79,6 +79,7 @@ if is_flashinfer_available(): if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner logger = logging.getLogger(__name__) @@ -299,10 +300,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): # Tree-mask scratch is fetched from the target backend only. self.is_draft_runner = model_runner.is_draft_worker - # [:n] view of a capture-stable buffer on the cuda-graph path; None on - # the eager path, which passes forward_batch.out_cache_loc through. - self._decode_kernel_loc: Optional[torch.Tensor] = None - self.cuda_graph_out_cache_loc_kernel: Optional[torch.Tensor] = None # Fused KV-scatter + q-concat on the decode dense-loc path (one launch # instead of set_mla_kv_buffer + concat_mla_absorb_q). Disabled under # async asserts: the fused path writes the pool directly and would @@ -411,6 +408,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): def _create_block_kv_indices( self, + plan: KVLocPlan, batch_size: int, max_blocks: int, req_pool_indices: torch.Tensor, @@ -430,11 +428,19 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): Returns: Block KV indices tensor """ + dcp_enabled = get_parallel().dcp_enabled + translated = self.kv_index_translator.is_translating and not dcp_enabled + # A translated table holds the plan's page ids and reads the sink past + # them: a draft extend pads every row's key length to its widest row, + # and the padded queries (whose outputs are dropped) read that far. block_kv_indices = torch.full( - (batch_size, max_blocks), -1, dtype=torch.int32, device=device + (batch_size, max_blocks), + 0 if translated else -1, + dtype=torch.int32, + device=device, ) - if get_parallel().dcp_enabled: + if dcp_enabled: self._fill_dcp_block_kv_indices( block_kv_indices, req_pool_indices, @@ -442,12 +448,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): ) return block_kv_indices - if self.kv_index_translator.is_translating: - self.kv_index_translator.fill_read_table( - out=block_kv_indices, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - ) + if translated: + self.kv_index_translator.copy_page_table(plan, out=block_kv_indices) else: create_flashmla_kv_indices_triton[ ( @@ -480,13 +482,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): self.decode_cuda_graph_kv_indices = torch.full( (max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device ) - # Unified pool: capture-stable buffer for the physical KV write loc, filled - # out-of-graph in init_forward_metadata_out_graph so the in-graph - # set_mla_kv_buffer captures no translate. - if self.kv_index_translator.is_translating: - self.cuda_graph_out_cache_loc_kernel = torch.zeros( - max_num_tokens, dtype=torch.int64, device=self.device - ) num_tokens_per_req = max_num_tokens // max_bs if is_float4_e2m1fn_x2(self.data_type): @@ -603,6 +598,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): def _apply_cuda_graph_metadata( self, + plan: KVLocPlan, bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, @@ -642,10 +638,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): # Update block indices for new sequences. if self.kv_index_translator.is_translating: - self.kv_index_translator.fill_read_table( - out=metadata.block_kv_indices, - req_pool_indices=req_pool_indices[:bs], - seq_lens=seq_lens, + # The plan's table reaches a verify's draft tail. + self.kv_index_translator.copy_page_table( + plan, out=metadata.block_kv_indices[:bs] ) else: create_flashmla_kv_indices_triton[ @@ -755,6 +750,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): forward_batch.seq_lens.device, ) self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=forward_batch.req_pool_indices, seq_lens=forward_batch.seq_lens, @@ -762,56 +758,13 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): ) else: self._apply_cuda_graph_metadata( + plan=forward_batch.kv_loc_plan, bs=bs, req_pool_indices=forward_batch.req_pool_indices, seq_lens=forward_batch.seq_lens, forward_mode=forward_mode, ) - # Out-of-graph on capture AND every replay-prep, so the in-graph - # set_mla_kv_buffer captures no translate. - if self.kv_index_translator.is_translating and ( - forward_mode.is_decode_or_idle() or forward_mode.is_target_verify() - ): - # The captured kernel consumes the whole buffer, so the tail a - # shorter replay leaves must go to slot 0 rather than live pages. - self._decode_kernel_loc = self.kv_index_translator.fill_capture_write_loc( - out=self.cuda_graph_out_cache_loc_kernel, - forward_batch=forward_batch, - width=self.cuda_graph_out_cache_loc_kernel.numel(), - ) - else: - self._decode_kernel_loc = None - - def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc: - """The loc an unfused KV scatter must write at: the capture-stable - buffer under a captured unified-pool decode, since the translate - rebinds `out_cache_loc` to a fresh tensor the graph never recorded; - the batch's own loc everywhere else. - """ - if self._decode_kernel_loc is not None: - # Filled by `fill_capture_write_loc`, which translates it. - return KVWriteLoc(self._decode_kernel_loc, physical=True) - return KVWriteLoc.for_batch(forward_batch) - - def _resolve_fused_write_loc( - self, forward_batch: ForwardBatch - ) -> Optional[torch.Tensor]: - """Write loc for the fused fp8 KV scatter, or None when this batch is - not covered by it. - - Captured decode refills `_decode_kernel_loc` out of the graph, and the - captured kernel must read that buffer. Eager decode on a unified pool - has no such buffer, and the caller falls back to the unfused path. - """ - if self._decode_kernel_loc is not None: - return self._decode_kernel_loc - return ( - None - if self.kv_index_translator.is_translating - else forward_batch.out_cache_loc - ) - def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize the metadata for a forward pass.""" if self._kv_shard_pool is not None: @@ -821,7 +774,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): forward_batch, ) - self._decode_kernel_loc = None # Delegate to parent for non-decode modes. if ( forward_batch.forward_mode.is_extend() @@ -912,6 +864,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): max_seqlen_pad = self._calc_padded_blocks(max_seq) block_kv_indices = self._create_block_kv_indices( + forward_batch.kv_loc_plan, bs, max_seqlen_pad, forward_batch.req_pool_indices, @@ -1287,7 +1240,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): assert q_rope is not None and k_rope is not None if cos_sin_cache is None: if save_kv_cache and self._fused_set_kv_concat_q_fp8: - loc = self._resolve_fused_write_loc(forward_batch) + loc = forward_batch.out_cache_loc if loc is not None: # Fused: bf16->fp8 quantize + KV scatter + q concat # in one launch; None when not covered. @@ -1323,42 +1276,23 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): assert k is not None and k_rope is not None, ( "For populating trtllm_mla kv cache, both k_nope and k_rope should be not None." ) - if self._decode_kernel_loc is not None: - if merge_query and self._fused_set_kv_concat_q: - # Fused: KV scatter + [q_nope | q_rope] concat in one - # launch; None when the inputs are not covered. - query = self._set_kv_and_concat_q_fused( - layer=layer, - loc=self._decode_kernel_loc, - k=k, - k_rope=k_rope, - q=q, - q_rope=q_rope, - ) - if query is None: - self.token_to_kv_pool.set_mla_kv_buffer( - layer, self._kv_write_loc(forward_batch), k, k_rope - ) - else: - # eager (or static pool): out_cache_loc is physical. - if ( - merge_query - and self._fused_set_kv_concat_q - and not self.kv_index_translator.is_translating - ): - # Static pool only, conservatively. - query = self._set_kv_and_concat_q_fused( - layer=layer, - loc=forward_batch.out_cache_loc, - k=k, - k_rope=k_rope, - q=q, - q_rope=q_rope, - ) - if query is None: - self.token_to_kv_pool.set_mla_kv_buffer( - layer, KVWriteLoc.for_batch(forward_batch), k, k_rope - ) + # `out_cache_loc` is physical on every pool, eager or captured + # (the runner's slot). + if merge_query and self._fused_set_kv_concat_q: + # Fused: KV scatter + [q_nope | q_rope] concat in one launch; + # None when the inputs are not covered. + query = self._set_kv_and_concat_q_fused( + layer=layer, + loc=forward_batch.out_cache_loc, + k=k, + k_rope=k_rope, + q=q, + q_rope=q_rope, + ) + if query is None: + self.token_to_kv_pool.set_mla_kv_buffer( + layer, KVWriteLoc.for_batch(forward_batch), k, k_rope + ) # Prepare query tensor inline (already built when the fused save-KV # path ran) @@ -1515,7 +1449,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): assert q_rope is not None and k_rope is not None if cos_sin_cache is None: if save_kv_cache and self._fused_set_kv_concat_q_fp8: - loc = self._resolve_fused_write_loc(forward_batch) + loc = forward_batch.out_cache_loc if loc is not None: # Fused: bf16->fp8 quantize + KV scatter + q concat # in one launch; None when not covered. @@ -1551,7 +1485,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): "For populating trtllm_mla kv cache, both k_nope and k_rope should be not None." ) self.token_to_kv_pool.set_mla_kv_buffer( - layer, self._kv_write_loc(forward_batch), k, k_rope + layer, KVWriteLoc.for_batch(forward_batch), k, k_rope ) # TODO refactor to avoid code duplication diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index e5c3bc36a8..0b134976fc 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -604,6 +604,7 @@ class TpModelWorker(BaseTpWorker): skip_attn_backend_init: Optional[bool] = None, # deprecated *, capture_hidden_mode: Optional[CaptureHiddenMode] = None, + return_kv_loc_plan: bool = False, ) -> GenerationBatchResult: # Get forward batch from schedule batch if batch is not None: @@ -649,6 +650,7 @@ class TpModelWorker(BaseTpWorker): expert_distribution_metrics=out.expert_distribution_metrics, routed_experts_output=out.routed_experts_output, indexer_topk_output=out.indexer_topk_output, + kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None, ) capture_pre_sample_logits(batch, forward_batch, logits_output) diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 90c7ba9fe2..320a3c69ac 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -26,6 +26,7 @@ from sglang.srt.utils.common import async_d2h as _async_d2h if TYPE_CHECKING: from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput from sglang.srt.managers.scheduler import GenerationBatchResult + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.speculative.spec_info import SpecInput @@ -118,6 +119,11 @@ class GenerationBatchResult: # until the accepted path comes back over the relay. spec_verify_out_cache_loc: Optional[torch.Tensor] = None + # The plan the forward wrote its KV through, when the caller asked for it + # (`return_kv_loc_plan`): a speculative worker's own writes into the same + # slots take their ids from it. + kv_loc_plan: Optional[KVLocPlan] = None + # PP+spec: [bs, spec_steps + 1] global node indices of the accepted path. # Every stage holds the KV for its own layers, so every stage has to compact # that path into its committed prefix; only the last stage can compute it. diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 6622a61c85..21975537ff 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -510,10 +510,9 @@ class KVCacheConfigurator: token_to_kv_pool = None # Unified-pool fast path: build req_to_token + token_to_kv pool + allocator - # from one byte buffer, then return. Gated to the target worker by ROLE: - # a compact-window DFLASH draft also passes req_to_token_pool=None, and - # would allocate a SECOND unified byte buffer here. Supports hybrid - # Mamba and hybrid SWA (not DSV4). + # from one byte buffer, then return. Target worker only: a compact-window + # DFLASH draft also passes req_to_token_pool=None. Supports hybrid Mamba + # and hybrid SWA (not DSV4). if ( get_memory().enable_unified_memory and req_to_token_pool is None @@ -581,7 +580,6 @@ class KVCacheConfigurator: alloc = token_to_kv_pool_allocator placement = self._fused_draft_from_target_buffer(alloc) if placement is not None: - # FUSED arm from sglang.srt.mem_cache.layout.fused_draft import ( draft_swa_layer_ids, ) @@ -619,7 +617,6 @@ class KVCacheConfigurator: token_to_kv_pool_allocator=alloc, unified_memory_pool=None, ) - # PRIVATE arm draft_virtual_id_space = alloc.draft_virtual_id_space assert draft_virtual_id_space >= sizes.max_total_num_tokens, ( "unified allocator virtual space smaller than the token " @@ -627,7 +624,7 @@ class KVCacheConfigurator: f"max_total_num_tokens={sizes.max_total_num_tokens}" ) # Round UP to page alignment (paged draft backends view the - # pool as (-1, page_size, H, D); the space is not aligned). + # pool as (-1, page_size, H, D); the virtual space is not aligned). page = max(int(self.pool_page_size or 1), 1) draft_virtual_id_space = ( (draft_virtual_id_space + page - 1) // page * page @@ -733,11 +730,8 @@ class KVCacheConfigurator: ) def _fused_draft_from_target_buffer(self, alloc): - """The placement this draft binds to, or None for the private-pool - fallback (the draft then owns a raw-virtual-indexed pool of its own). - Target boot declines a placement whenever the fused region cannot - serve the draft (`_fused_draft_decision`), so the private arm is the - rollback lever, not a boot-order bug.""" + """The placement this draft binds to, or None when the target placed + none and the draft builds a private pool over the virtual id space.""" if not ( self.spec_algorithm.is_eagle() or self.spec_algorithm.is_dflash_family() ): @@ -961,14 +955,10 @@ class KVCacheConfigurator: def _fused_draft_decision(self): """Whether, and where, the draft's layers fuse into the target's - sub-pools. Fusion applies only for: unified memory ON, a unified target - (hybrid-SWA or mamba hybrid, either full-pool kind), an EAGLE-family - or DFLASH/DSPARK algorithm whose draft config was loaded at target - boot. It declines, keeping the draft's private pool, for a non-MHA - draft, a draft backend off the translated MHA rails, DCP, HiCache, - host-pool decode retraction, or an explicit draft KV dtype unlike the - host's; `place_fused_draft` then admits or declines the draft's layer - kinds.""" + sub-pools. An empty decision means fusion does not apply (unified + memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config + loaded at target boot); a declined one says why the draft cannot fuse. + The pool factories decide whether a decline refuses the boot.""" from sglang.srt.mem_cache.layout.fused_draft import ( FusedDraftDecision, draft_kv_profile, @@ -998,10 +988,9 @@ class KVCacheConfigurator: "the fused region holds dense MHA K/V rows" ) ) - # The backend(s) the draft runner will run. A DFLASH-family draft - # worker resolves one as `build_draft_tp_worker` does; an EAGLE draft - # runs the published draft backend, else the target's prefill/decode - # pair. A published draft backend carries model-hook declarations. + # The draft runner's backend, resolved as its worker will: an EAGLE + # draft with no backend of its own runs the target's prefill/decode + # pair. The published draft backend carries model-hook declarations. if self.spec_algorithm.is_dflash_family(): from sglang.srt.speculative.draft_worker_common import ( resolve_draft_worker_attention_backend, @@ -1066,10 +1055,9 @@ class KVCacheConfigurator: else 1 ) # A fused draft stores its rows in the host's KV dtype. An explicit - # draft dtype resolves without the draft model, as the draft runner - # resolves it, so a mismatch is declined here; `auto` follows the - # draft's quant config, which only the draft runner knows, and is - # checked when it binds. + # draft dtype resolves here as the draft runner will; `auto` follows + # the draft's quant config, which only the draft runner knows, so it + # is checked when the draft binds. draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype if draft_kv_cache_dtype not in (None, "auto"): from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype @@ -1098,11 +1086,10 @@ class KVCacheConfigurator: ) def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]: - """Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft + - pad) for the boot solve's cell model. Single source of truth: assembled - through the same spec the pool factory builds, so the priced entry and - the allocated entry cannot drift. None when the draft does not fuse - into that sub-pool.""" + """Per-token bytes of ``sub_pool_name``'s fused entry (host + draft + + pad), built from the same spec as the pool factory's so the priced and + allocated entries agree; None when the draft does not fuse into that + sub-pool.""" placement = self._fused_draft_decision().placement if placement is None or sub_pool_name != "full": return None @@ -1150,14 +1137,11 @@ class KVCacheConfigurator: def _fused_draft_for_mamba_factory(self): """`_fused_draft_for_pool_factory` for a mamba-hybrid host. Its unified - buffer takes the WHOLE profiled KV budget (`unified_total_bytes`), so a - private draft pool -- sized to the full virtual id span -- would be - allocated on top of it, unbudgeted. Until the mamba solve prices that - pool, an EAGLE-family or DFLASH draft that does not fuse is refused here - instead of overcommitting GPU memory. A DSPARK draft that does not fuse - keeps its private pool, unpriced: refusing it would refuse DSPARK on - these hosts whenever its draft declines fusion, as the Kimi-Linear - default (a trtllm_mha draft) does.""" + buffer takes the whole profiled KV budget, so a private draft pool + would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that + does not fuse is refused. A DSPARK draft that does not fuse keeps its + private pool, unpriced, so DSPARK still boots where its draft declines + fusion (as the Kimi-Linear default, a trtllm_mha draft, does).""" decision = self._fused_draft_decision() placement = self._fused_draft_for_pool_factory(decision) if ( @@ -1169,18 +1153,16 @@ class KVCacheConfigurator: raise ValueError( "--enable-unified-memory + EAGLE/EAGLE3/DFLASH on a mamba-hybrid " "target needs the draft's KV fused into the target's pages, but " - f"this draft would keep a private pool ({reason}). A mamba host's " - "unified buffer takes the whole KV budget, so a private draft " - "pool would overcommit GPU memory." + f"this draft cannot fuse ({reason}). A private draft pool would " + "overcommit the KV budget the unified buffer already takes." ) return placement def _fused_draft_for_swa_factory(self): """`_fused_draft_for_pool_factory` for a hybrid-SWA host. Its boot solve prices a private EAGLE draft pool at the target's per-token size, not - at the draft's own heads, head_dim and KV dtype, so a draft that does - not fuse is refused here, as on a mamba host, instead of overcommitting - GPU memory.""" + the draft's own geometry and KV dtype, so an EAGLE draft that does not + fuse is refused rather than overcommit GPU memory.""" decision = self._fused_draft_decision() placement = self._fused_draft_for_pool_factory(decision) if ( @@ -1192,15 +1174,14 @@ class KVCacheConfigurator: raise ValueError( "--enable-unified-memory + EAGLE/EAGLE3 on a hybrid-SWA target " "needs the draft's KV fused into the target's pages, but this " - f"draft would keep a private pool ({reason}), which the unified " - "pool's sizing does not price." + f"draft cannot fuse ({reason}), and the unified pool's sizing " + "does not price a private draft pool." ) return placement def _fused_draft_for_pool_factory(self, decision): - """The placement a pool factory is handed, from ONE resolved decision, - so the boot log reports exactly that placement: a declined fusion and - an engaged one otherwise look identical from outside.""" + """The placement a pool factory is handed. It logs the decision it + returns, so the boot log shows whether fusion engaged or why not.""" if decision.placement is None: if decision.declined is not None: logger.warning("fused draft KV disabled: %s", decision.declined) diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 6b8bffd7d2..69b2109f91 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -23,18 +23,29 @@ A KV slot can be named in two id spaces: The two coincide on a plain pool, so nothing here does any work there. -Backends get a `KVIndexTable`, which answers "what do I gather from, and -which row is mine?": +A pool can hold several sub-pools (full attention, sliding window), each with +its own physical ids and named by an `IdSpace` (`space`). Writes are translated +once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly +once, by the readers below and nowhere else; a backend never translates. The +plan's read table exists only once a table reader asks for it (`read_table`, +`copy_page_table`), built into that reader's own capture-stable table when it +is the first; later readers take it, earlier ones translate in their own +gather. The readers answer "what do I gather from, and which row is mine?" +with a `KVIndexTable`: ids[row_ids[b], pos] - plain pool : ids = req_to_token, row_ids = req_pool_indices (those very - objects - no copy, no kernel) - unified : ids = a built array of physical ids, - row_ids = arange(batch_size) + plain pool : ids = req_to_token, row_ids = req_pool_indices (those very + objects - no copy, no kernel) + shared table : ids = the plan's array of physical page ids, + row_ids = arange(batch_size) + no table : ids = req_to_token, row_ids = req_pool_indices, and + v2p = the sub-pool's page table, applied by the gather Backends call their own copy a *page table* (fa3) or a *block table* -(trtllm); here it is the **index table**. +(trtllm); here it is the **index table**. `read_source` serves a gather +kernel, `pack_read_stream` a CSR stream, and `read_table` / `copy_page_table` +a page-table kernel. Converting only ever rewrites the page number and keeps the in-page offset, so one page-granular table serves both kinds of consumer: a block-table backend @@ -42,16 +53,15 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as token_id = entry * entry_page_size + pos % entry_page_size -WRITES, IN TWO PHASES. The full-side write loc is rebound to physical -ids at ForwardBatch construction - the earliest consumer can snapshot it -right after. The sliding-window write loc is derived at the same moment as -read table, into the same index table. +The one translation outside a plan is DCP's read production +(`translate_dcp_read_ids`): which ids a rank reads is only known where the +DCP index kernels select them. """ from __future__ import annotations -import weakref -from typing import Optional, Tuple +import functools +from typing import Dict, Optional import msgspec import torch @@ -63,6 +73,7 @@ from sglang.kernels.ops.kvcache.kv_read_table import ( build_kv_read_table, build_kv_read_table_packed, ) +from sglang.srt.layers.dcp.layout import localize_dcp_indices from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWAAllocatorBase, ) @@ -71,30 +82,36 @@ from sglang.srt.mem_cache.allocator.unified_mamba import ( ) from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.kv_loc_plan import ( + IdSpace, + IdSpaceKind, + KVLocPlan, + pad_with_sink, + window_read_extent, +) from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils import is_npu - -class KVReadTables(msgspec.Struct, frozen=True): - """One capture-stable destination, however many id spaces the pool has. - - A backend holds this and hands it back to `build_index_table(into=...)`; - it never has to know whether there is a sliding-window space behind it. - """ - - full: torch.Tensor - sliding_window: Optional[torch.Tensor] +_is_npu = is_npu() class KVIndexTable(msgspec.Struct, frozen=True): - """Collection of what one batch gathers from.""" + """What one batch gathers from, and how its entries become the ids a + kernel reads: lane ``b``'s entry ``j`` is ``ids[row_ids[b] * row_stride + + j // entry_page_size]``, expanded to tokens when an entry is a page, then + mapped through ``v2p`` (page-granular, at the translator's page size) when + one is given. A gather kernel executes exactly this; it never decides + whether or how to translate.""" ids: torch.Tensor # 2-D array of KV ids to gather from row_ids: torch.Tensor # which row belongs to batch lane b row_stride: int # stride between rows of `ids`, in elements entry_page_size: int # what one entry covers: 1 = a token, N = a page of N - is_translated: bool # entries are already physical ids - sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array + is_translated: bool # the ids a gather yields are physical + # Virtual rows a gather translates on the way out (no shared table this + # iteration); None when the entries are already what the kernel reads. + v2p: Optional[torch.Tensor] = None class KVIndexTranslator: @@ -108,6 +125,7 @@ class KVIndexTranslator: token_to_kv_pool, page_size: int, device: str, + is_draft_worker: bool = False, ): self.req_to_token = req_to_token self.page_size = page_size @@ -125,55 +143,104 @@ class KVIndexTranslator: host_allocator is not None and host_allocator is token_to_kv_pool_allocator ) self.is_translating = is_unified_target or is_fused_draft + self._spaces: Dict[IdSpaceKind, IdSpace] = {} if self.is_translating: alloc = token_to_kv_pool_allocator self._capture_page_size = alloc.page_size self._full_v2p_table = alloc.full_v2p_page_table - self._full_p2v_table = alloc.full_p2v_page_table self._translate_full = alloc.translate_kv_loc - # The WRITE loc is the one id that arrives DCP-WIDENED: read indices - # are collapsed by the DCP index kernels, `out_cache_loc` still - # carries the owner rule in `loc % dcp_size`. Identity with the read - # translate when dcp_size == 1. - self._translate_write_full = alloc.translate_write_loc # DCP read ids stay WIDENED to the consumer: selecting this rank's # share changes the length, so only the production site can do it. self.defer_read_translate = get_parallel().attn_dcp_size > 1 + # Keyed by the allocator, which owns the tables: a fused draft's + # runner shares the target's full sub-pool, and so its key. + self._spaces[IdSpaceKind.FULL] = IdSpace( + key=(IdSpaceKind.FULL, id(alloc)), + # The WRITE loc is the one id that arrives DCP-WIDENED: read + # indices are collapsed by the DCP index kernels, `out_cache_loc` + # still carries the owner rule in `loc % dcp_size`. + write=alloc.translate_write_loc, + read_v2p=None if self.defer_read_translate else self._full_v2p_table, + ) routes_window_layers = is_unified_target or isinstance( token_to_kv_pool, BaseSWAKVPool ) if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers: - self._swa_v2p_table = alloc.swa_v2p_page_table - self._swa_write_loc_from_full = self._swa_write_loc_unified - else: - self._swa_v2p_table = None - self._swa_write_loc_from_full = None + # Straight from the virtual ids through the sliding-window + # sub-pool's own table. + self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace( + key=(IdSpaceKind.SLIDING_WINDOW, id(alloc)), + write=alloc.translate_loc_from_full_to_swa, + read_v2p=( + None if self.defer_read_translate else alloc.swa_v2p_page_table + ), + ) else: self._capture_page_size = page_size self._full_v2p_table = None - self._full_p2v_table = None self._translate_full = None - self._translate_write_full = None self.defer_read_translate = False - self._swa_v2p_table = None + parallel = get_parallel() + if _is_npu and parallel.dcp_enabled and not is_draft_worker: + # An NPU DCP target writes its rank's share of the window in + # rank-local slots (-1: another rank's); the scheduler, + # `req_to_token` and the replicated draft pool keep the + # allocator-global ones the window is planned in. + localize = functools.partial( + localize_dcp_indices, + dcp_size=parallel.dcp_size, + dcp_rank=parallel.dcp_rank, + interleave_size=page_size, + ) + self._spaces[IdSpaceKind.FULL] = IdSpace( + key=(IdSpaceKind.FULL, "dcp-rank-local"), write=localize + ) + else: + localize = None + self._spaces[IdSpaceKind.FULL] = IdSpace(key=(IdSpaceKind.FULL, None)) # `translate_loc_from_full_to_swa` is abstract on `BaseSWAKVPool`, # which is also what the backends' `_resolve_swa_kv_pool` keys on. - self._swa_write_loc_from_full = ( - token_to_kv_pool.translate_loc_from_full_to_swa - if isinstance(token_to_kv_pool, BaseSWAKVPool) - and ( - not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - or token_to_kv_pool.request_window is None + # Reads stay in `req_to_token`; the backends map them themselves. + if isinstance(token_to_kv_pool, BaseSWAKVPool) and ( + not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) + or token_to_kv_pool.request_window is None + ): + to_swa = token_to_kv_pool.translate_loc_from_full_to_swa + self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace( + key=(IdSpaceKind.SLIDING_WINDOW, id(token_to_kv_pool)), + write=( + to_swa + if localize is None + else lambda ids: to_swa(localize(ids)) + ), ) - else None - ) self._rows: Optional[torch.Tensor] = ( torch.arange(req_to_token.shape[0], dtype=torch.int64, device=device) if self.is_translating else None ) - self._index_table_memo: Optional[Tuple[weakref.ref, KVIndexTable]] = None + + def space(self, kind: IdSpaceKind) -> Optional[IdSpace]: + """How this runner's ``kind`` sub-pool names the iteration's slots, or + None when its pool has no such sub-pool.""" + return self._spaces.get(kind) + + def write_ids( + self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL + ) -> Optional[torch.Tensor]: + """The ids ``forward_batch`` writes in this runner's ``kind`` sub-pool, + taken from its plan and padded with the sink to its `out_cache_loc`'s + length (a padded batch, a captured graph's buffer). None when there is + no write loc or no such sub-pool.""" + plan = getattr(forward_batch, "kv_loc_plan", None) + out_cache_loc = getattr(forward_batch, "out_cache_loc", None) + if plan is None or out_cache_loc is None: + return None + ids = plan.write_ids( + self, kind=kind, cols=getattr(forward_batch, "kv_loc_cols", None) + ) + return pad_with_sink(ids, out_cache_loc.shape[0]) def capture_token_capacity(self, max_token_pool_size: int) -> int: """Host capture rows are indexed by request-token IDs, not kernel IDs. @@ -185,240 +252,305 @@ class KVIndexTranslator: return self._full_v2p_table.numel() * self._capture_page_size return max_token_pool_size + self.page_size - def full_flat_v2p(self) -> Optional[torch.Tensor]: - """The full-side v2p page table for a kernel that translates flat ids - itself, or ``None`` when this runner does not translate (pass-through - and static pools).""" - if not self.is_translating: - return None - return self._full_v2p_table + # -- the iteration's plan -------------------------------------------------- - # -- per-batch view -------------------------------------------------------- + def plan( + self, + *, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + write_virtual: Optional[torch.Tensor], + read_extent: int = 0, + write_slots: Optional[torch.Tensor] = None, + ) -> KVLocPlan: + """This iteration's ids, translated once. ``write_virtual`` is the + iteration's write window (``[batch, window]`` flattened when its + forwards write columns of it); ``read_extent`` is how far past + ``seq_lens`` the iteration's reads reach. ``write_slots`` replaces + ``write_virtual`` for a runner's own buffer (`bind_runner_slots`).""" + return KVLocPlan( + source=self, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=write_virtual, + read_extent=read_extent, + write_slots=write_slots, + ) + + def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan: + """The plan of a forward that is its own iteration, from its own + fields: its ``out_cache_loc`` is the write window (virtual, or with + ``runner_slots`` a runner's own buffer), and its reads reach + `window_read_extent` past its lengths.""" + write_ids = forward_batch.out_cache_loc + seq_lens = getattr(forward_batch, "seq_lens", None) + seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None) + encoder_lens = getattr(forward_batch, "encoder_lens", None) + if ( + encoder_lens is not None + and seq_lens is not None + and self.reads_are_translated + ): + # An encoder-decoder row holds its encoder tokens ahead of the + # decoder's, and its self-attention reads past `seq_lens`. + seq_lens = seq_lens + encoder_lens + encoder_lens_cpu = getattr(forward_batch, "encoder_lens_cpu", None) + seq_lens_cpu = ( + None + if seq_lens_cpu is None or encoder_lens_cpu is None + else seq_lens_cpu + torch.tensor(encoder_lens_cpu).to(seq_lens_cpu) + ) + return self.plan( + req_pool_indices=getattr(forward_batch, "req_pool_indices", None), + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=None if runner_slots else write_ids, + write_slots=write_ids if runner_slots else None, + read_extent=window_read_extent( + getattr(forward_batch, "forward_mode", None), + getattr(forward_batch, "spec_info", None), + write_ids, + getattr(forward_batch, "batch_size", 0), + ), + ) + + def bind_runner_slots(self, forward_batch) -> None: + """Bind a batch a runner builds over its own buffers, to capture a + graph or to warm up with. Its write ids are the runner's buffer, used + as it is; its reads plan over the runner's own rows and lengths.""" + self.own_plan(forward_batch, runner_slots=True).bind(forward_batch, self) + + def build_iteration_table( + self, + plan: KVLocPlan, + space: IdSpace, + *, + rows: Optional[int], + previous: Optional[KVIndexTable], + into: Optional[torch.Tensor] = None, + ) -> KVIndexTable: + """Build ``plan``'s page table in ``space`` for `KVLocPlan.read_table`: + the passthrough where reads stay virtual, else one build over ``[0, + seq_lens + read_extent)``. Rows past the plan's batch read the sink; a + request wider than ``previous`` copies its rows rather than translating + again. ``into``, the asking reader's capture-stable table, is built in + place when wide enough and then serves as the plan's table.""" + if space.read_v2p is None: + return self._passthrough_table(plan.req_pool_indices) + bs = int(plan.req_pool_indices.numel()) + rows = max(bs, rows or 0) + if previous is not None: + out = torch.zeros( + (rows, previous.ids.shape[1]), dtype=torch.int32, device=self.device + ) + out[: previous.ids.shape[0]].copy_(previous.ids) + else: + row_pages = -(-self.req_to_token.shape[1] // self.page_size) + slc = plan.seq_lens_cpu + if slc is not None and slc.numel() > 0: + max_seq = int(slc.max()) + plan.read_extent + width = min(max(-(-max_seq // self.page_size), 1), row_pages) + elif into is not None: + # No host lengths: a row's speculative headroom outruns the + # captured table, which is as wide as its graph reads. + width = min(into.shape[1], row_pages) + else: + width = row_pages + # `zero_tail` writes the sink past each live prefix, so neither a + # reused `into` nor an `empty` table keeps stale columns. + if into is not None and into.shape[0] >= rows and into.shape[1] >= width: + out = into[:rows, :width] + else: + out = torch.empty((rows, width), dtype=torch.int32, device=self.device) + if rows > bs: + out[bs:].zero_() + build_kv_read_table( + req_to_token=self.req_to_token, + req_pool_indices=plan.req_pool_indices, + seq_lens=plan.seq_lens, + v2p=space.read_v2p, + page_size=self.page_size, + max_pages=width, + out=out, + zero_tail=True, + seq_len_delta=plan.read_extent, + ) + return KVIndexTable( + ids=out, + row_ids=self._rows[:rows], + row_stride=out.stride(0), + entry_page_size=self.page_size, + is_translated=True, + ) + + # -- readers --------------------------------------------------------------- + + def _reads_translated(self, kind: IdSpaceKind) -> bool: + space = self._spaces.get(kind) + return space is not None and space.read_v2p is not None + + def read_source( + self, + plan: KVLocPlan, + *, + req_pool_indices: torch.Tensor, + bs: int, + kind: IdSpaceKind = IdSpaceKind.FULL, + ) -> KVIndexTable: + """The row source a gather kernel reads lane ``b``'s ``kind`` ids from: + the plan's table once a table reader has had it built, else the virtual + `req_to_token` rows with ``v2p`` for the kernel's gather to apply (no + table is built for it). The plain `req_to_token` passthrough where + reads stay virtual (a static pool, or DCP).""" + if not self._reads_translated(kind): + return self._passthrough_table(req_pool_indices) + if plan.has_read_table(kind): + return self._plan_table(plan, kind=kind, rows=bs) + return KVIndexTable( + ids=self.req_to_token, + row_ids=req_pool_indices, + row_stride=self.req_to_token.stride(0), + entry_page_size=1, + is_translated=True, + v2p=self.space(kind).read_v2p, + ) + + def read_table( + self, + plan: KVLocPlan, + *, + kind: IdSpaceKind = IdSpaceKind.FULL, + rows: Optional[int] = None, + ) -> KVIndexTable: + """The plan's ``kind`` page table, exactly ``rows`` rows when given + (lanes past the plan's batch read the sink); the ``req_to_token`` + passthrough where reads stay virtual (a static or private pool, or DCP, + where the producing kernel selects this rank's share).""" + if self._reads_translated(kind): + return self._plan_table(plan, kind=kind, rows=rows) + return self._passthrough_table(plan.req_pool_indices) + + def _plan_table( + self, + plan: KVLocPlan, + *, + kind: IdSpaceKind, + rows: Optional[int] = None, + into: Optional[torch.Tensor] = None, + ): + assert plan.is_read_by(self), ( + "a translating reader must read through the plan of its own " + "req_to_token rows" + ) + return plan.read_table(kind=kind, rows=rows, into=into) + + def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable: + return KVIndexTable( + ids=self.req_to_token, + row_ids=req_pool_indices, + row_stride=self.req_to_token.stride(0), + entry_page_size=1, + is_translated=False, + ) + + def pack_read_stream( + self, + plan: KVLocPlan, + *, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + indptr: torch.Tensor, + out: torch.Tensor, + kv_start_idx: Optional[torch.Tensor] = None, + kind: IdSpaceKind = IdSpaceKind.FULL, + token_mapping: Optional[torch.Tensor] = None, + ) -> bool: + """Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans + over; return whether they are physical. Packed from the plan's table + once a table reader has had it built, else gathered and translated + straight into ``out``. ``False`` means still-VIRTUAL full-attention ids + (a static pool, or DCP) for the caller to finish; a static SWA pool can + fuse its full->swa table into the gather as ``token_mapping``.""" + bs = int(seq_lens.numel()) + if self._reads_translated(kind) and not plan.has_read_table(kind): + assert plan.is_read_by(self), ( + "a translating reader must read through the plan of its own " + "req_to_token rows" + ) + # `seq_lens` sizes the batch: a caller may hold a wider + # req_pool_indices (a padded graph buffer), whose extra lanes have + # no length to bound. + assert req_pool_indices.numel() >= bs, ( + f"pack_read_stream: {bs} stream lanes, " + f"{req_pool_indices.numel()} request rows" + ) + build_kv_read_table_packed( + req_to_token=self.req_to_token, + req_pool_indices=req_pool_indices[:bs], + seq_lens=seq_lens, + v2p=self.space(kind).read_v2p, + indptr=indptr, + page_size=self.page_size, + max_tokens=out.numel(), + out=out, + kv_start_idx=kv_start_idx, + ) + return True + # The plan's table or a passthrough: the entries are the stream's ids. + src = self.read_source( + plan, req_pool_indices=req_pool_indices, bs=bs, kind=kind + ) + assert src.v2p is None, "pack_read_stream: a table-less translating read" + assert token_mapping is None or not src.is_translated + create_flashinfer_kv_indices_triton[(bs,)]( + src.ids, + src.row_ids[:bs], + seq_lens, + indptr, + kv_start_idx, + out, + src.row_stride, + ENTRY_PAGE_SIZE=src.entry_page_size, + token_mapping=token_mapping, + ) + return src.is_translated or token_mapping is not None + + def copy_page_table( + self, + plan: KVLocPlan, + *, + out: torch.Tensor, + kind: IdSpaceKind = IdSpaceKind.FULL, + ) -> None: + """Fill ``out``, a captured graph's page table (padded lanes read the + sink), with the plan's ``kind`` table. The plan's first table reader + builds straight into ``out``, which then serves as the plan's table; a + later one copies it. Columns past the plan's width keep stale values, + which kernels never read past their own lengths.""" + assert self._reads_translated(kind), ( + "copy_page_table: reads stay virtual here (a non-unified pool, or " + "DCP, where the caller selects this rank's share itself)" + ) + rows = out.shape[0] + first = not plan.has_read_table(kind) + table = self._plan_table( + plan, kind=kind, rows=rows, into=out if first else None + ) + if table.ids.data_ptr() == out.data_ptr(): + return # built in place, or this very buffer is the plan's table + width = min(out.shape[1], table.ids.shape[1]) + out[:, :width].copy_(table.ids[:rows, :width]) + + # -- dispositions ---------------------------------------------------------- @property def reads_are_translated(self) -> bool: """Whether a read this translator fills comes out physical. False on a non-unified pool, and under DCP, where the ids stay VIRTUAL for ``translate_dcp_read_ids`` to finish.""" - return self.is_translating and not self.defer_read_translate - - def fill_packed_read_stream( - self, - *, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - indptr: torch.Tensor, - total_tokens: int, - out: torch.Tensor, - kv_start_idx: Optional[torch.Tensor] = None, - sliding_window: bool = False, - token_mapping: Optional[torch.Tensor] = None, - ) -> bool: - """Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and - report whether they came out translated. - - Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``. - Unified: one fused gather-and-translate, so no caller needs a - ``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per - resident token, a length the pool bounds. - - ``sliding_window`` selects the swa sub-pool's own id space, built from - VIRTUAL ids and never chained through full-physical. A ``False`` return - means the ids are still VIRTUAL: the DCP path defers translation to - ``translate_dcp_read_ids``. A static SWA pool can pass its full->swa - table as ``token_mapping`` to fuse that translation into the gather. - """ - # `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices - # (the padded graph buffer), and the extra lanes have no length to bound. - bs = int(seq_lens.numel()) - assert req_pool_indices.numel() >= bs, ( - f"fill_packed_read_stream: {req_pool_indices.numel()} req rows for " - f"{bs} lengths" - ) - req_pool_indices = req_pool_indices[:bs] - - if not self.reads_are_translated: - create_flashinfer_kv_indices_triton[(bs,)]( - self.req_to_token, - req_pool_indices, - seq_lens, - indptr, - kv_start_idx, - out, - self.req_to_token.stride(0), - ENTRY_PAGE_SIZE=1, - token_mapping=token_mapping, - ) - return token_mapping is not None - - if sliding_window: - assert self._swa_v2p_table is not None, ( - "fill_packed_read_stream: sliding_window on a pool with no swa sub-pool" - ) - build_kv_read_table_packed( - req_to_token=self.req_to_token, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - v2p=self._swa_v2p_table if sliding_window else self._full_v2p_table, - indptr=indptr, - page_size=self.page_size, - max_tokens=total_tokens, - out=out, - kv_start_idx=kv_start_idx, - ) - return True - - def build_index_table( - self, - *, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - max_pages: Optional[int] = None, - into: Optional[KVReadTables] = None, - seq_len_delta: int = 0, - ) -> KVIndexTable: - """The one per-batch entry point. - - Non-unified: the raw ``(req_to_token, req_pool_indices)`` passthrough, - no tensor ops and no copies. Unified: fills each row's live prefix and - returns the table WHOLE, so a caller needing a stable pointer (a - captured graph bakes it) passes its own tables in ``into``; - ``into=None`` allocates of width ``max_pages`` instead. - - ``seq_len_delta`` widens every row's live prefix -- the whole-sequence - verify contract (draft KV read back from the pool). An eager caller's - ``max_pages`` must already cover the delta. - """ - if not self.is_translating or self.defer_read_translate: - return KVIndexTable( - ids=self.req_to_token, - row_ids=req_pool_indices, - row_stride=self.req_to_token.stride(0), - entry_page_size=1, - is_translated=False, - sliding_window_ids=None, - ) - - bs = int(req_pool_indices.numel()) - if into is not None: - out_full = into.full - out_swa = into.sliding_window - # A caller-owned table may be padded wider than req_to_token's span - # (trtllm_mla / flashmla pad to a page-count bound); the columns - # past it have no source to read, so stop there. - width = min( - out_full.shape[1] if max_pages is None else max_pages, - -(-self.req_to_token.shape[1] // self.page_size), - ) - else: - assert max_pages is not None, ( - "KVIndexTranslator.build_index_table: allocating needs max_pages " - "(from the batch's seq_lens_cpu max)" - ) - width = max_pages - out_full = torch.zeros((bs, width), dtype=torch.int32, device=self.device) - out_swa = ( - torch.zeros((bs, width), dtype=torch.int32, device=self.device) - if self._swa_v2p_table is not None - else None - ) - - build_kv_read_table( - req_to_token=self.req_to_token, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - v2p=self._full_v2p_table, - page_size=self.page_size, - max_pages=width, - out=out_full, - seq_len_delta=seq_len_delta, - ) - if out_swa is not None: - build_kv_read_table( - req_to_token=self.req_to_token, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - v2p=self._swa_v2p_table, - page_size=self.page_size, - max_pages=width, - out=out_swa, - seq_len_delta=seq_len_delta, - ) - return KVIndexTable( - ids=out_full, - row_ids=self._rows[:bs], - row_stride=out_full.stride(0), - entry_page_size=self.page_size, - is_translated=True, - sliding_window_ids=out_swa, - ) - - def fill_read_table( - self, - *, - out: torch.Tensor, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - sliding_window_out: Optional[torch.Tensor] = None, - seq_len_delta: int = 0, - ) -> None: - """`build_index_table(into=...)` for a caller that owns a bare block - table rather than a KVReadTables: the page-table consumers read that - table directly, its rows already being the index table's rows. - - `sliding_window_out` fills the swa twin in the same pass, for a hybrid - model whose kernels take two block tables. - """ - # `reads_are_translated`, not `is_translating`: under DCP the builder - # returns the passthrough view and writes nothing, so `is_translating` - # would let a caller keep a stale table and never hear about it. - assert self.reads_are_translated, ( - "KVIndexTranslator.fill_read_table cannot fill a page table when " - "reads stay virtual (a non-unified pool, or DCP, where the caller " - "must select this rank's share itself)" - ) - assert sliding_window_out is None or self._swa_v2p_table is not None, ( - "KVIndexTranslator.fill_read_table: asked for a sliding-window " - "table on a pool with no swa sub-pool" - ) - self.build_index_table( - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - into=KVReadTables(full=out, sliding_window=sliding_window_out), - seq_len_delta=seq_len_delta, - ) - - def index_table_for_batch(self, forward_batch) -> KVIndexTable: - """Eager per-batch view, memoized in one slot keyed by batch identity - so multi-consumer metadata builds share a build. The next batch - replaces the slot; consumers only read during their own build. The - captured path does not memoize -- it refreshes its buffers per - replay.""" - memo = self._index_table_memo - if memo is not None and memo[0]() is forward_batch: - return memo[1] - max_pages = None - if self.is_translating: - # `seq_lens_cpu` is a non-None but STALE slice on a gpu_only - # batch; `seq_lens_sum` is the signal that it is live. A stale max - # under-sizes the table and the tail then reads as the sink. - slc = forward_batch.seq_lens_cpu - if ( - forward_batch.seq_lens_sum is not None - and slc is not None - and slc.numel() > 0 - ): - max_seq = int(slc.max()) - else: - max_seq = self.req_to_token.shape[1] - max_pages = max(-(-max_seq // self.page_size), 1) - view = self.build_index_table( - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=forward_batch.seq_lens, - max_pages=max_pages, - ) - self._index_table_memo = (weakref.ref(forward_batch), view) - return view + return self._reads_translated(IdSpaceKind.FULL) @property def full_v2p_table(self) -> Optional[torch.Tensor]: @@ -426,43 +558,11 @@ class KVIndexTranslator: pool needs no translation. For the DCP page-table builders, whose gather is over a rank's cyclic - slice rather than a row prefix, so `build_index_table` cannot serve - them. + slice rather than a row prefix, so a plan's row-prefix table cannot + serve them. """ return self._full_v2p_table - def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable: - """Whole-sequence-verify view: every row's live prefix widened by - `seq_len_delta` columns (the drafts are read back from the pool). - Not memoized -- verify is one consumer, and the batch's memoized - prefix table stays valid for the others. - - The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch - carries `seq_lens_cpu` already expanded by the window and keeps the - live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded - lens would count the window twice. The width never exceeds the - `req_to_token` row it is read from.""" - max_pages = None - if self.is_translating: - row_pages = -(-self.req_to_token.shape[1] // self.page_size) - live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None) - slc = live if live is not None else forward_batch.seq_lens_cpu - if ( - forward_batch.seq_lens_sum is not None - and slc is not None - and slc.numel() > 0 - ): - max_seq = int(slc.max()) + seq_len_delta - max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages) - else: - max_pages = row_pages - return self.build_index_table( - req_pool_indices=forward_batch.req_pool_indices, - seq_lens=forward_batch.seq_lens, - max_pages=max_pages, - seq_len_delta=seq_len_delta, - ) - def bind_and_verify_backends(self, backends) -> None: """Boot: make every reachable backend carry THIS translator. @@ -481,84 +581,13 @@ class KVIndexTranslator: "copy, not build its own." ) - # -- write loc (phase 1; phase 2 lives in build_index_table) ---------------- + def bind_own_plan(self, forward_batch) -> None: + """Bind a batch built outside `ForwardBatch.init_new` that is its own + iteration to a plan of its own fields (`own_plan`). A forward that + shares its iteration's window takes that plan instead.""" + self.own_plan(forward_batch).bind(forward_batch, self) - def rebind_write_loc(self, forward_batch) -> None: - """Phase 1 of the WRITE contract: translate the batch's write loc to - FULL-side physical ids exactly once, at ForwardBatch construction, and - mark it physical. On non-unified pools the allocation is already - physical, so only the mark is set. - - REBIND, never mutate: the translate returns a FRESH tensor, so the - ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept / - in-flight machinery that reads it. The pre-translate tensor stays on - the batch for `fill_capture_write_loc`. - """ - self._index_table_memo = None - if forward_batch.out_cache_loc is None: - return - if self.is_translating: - forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc - forward_batch.out_cache_loc = self._translate_write_full( - forward_batch.out_cache_loc - ) - forward_batch.out_cache_loc_is_physical = True - - def fill_capture_write_loc( - self, - *, - out: torch.Tensor, - forward_batch, - width: Optional[int] = None, - ) -> Optional[torch.Tensor]: - """Translate this batch's WRITE loc straight into ``out``, a backend's - capture-stable buffer, and return the live ``[:n]`` view. One launch - fills the live prefix and clears the tail a shorter replay leaves; - None when this pool needs no translation. - - Must run at metadata-init time: `out` is reused every step, so filling - it sooner would race a still-pending previous step under overlap - scheduling. - """ - if not self.is_translating: - return None - virtual = forward_batch.out_cache_loc_virtual - if virtual is None: - loc = forward_batch.out_cache_loc - if loc is None: - return None - # The runner builds the capture batch outside `init_new`, so no - # rebind marked its virtual source; bake it holding sink ids. - width = int(loc.numel()) if width is None else int(width) - out[:width].zero_() - return out[: int(loc.numel())] - n = int(virtual.numel()) - width = n if width is None else int(width) - buf = out[:width] - self._translate_write_full(virtual, out=buf, out_width=width) - return buf[:n] - - def sliding_window_write_loc_for( - self, out_cache_loc: Optional[torch.Tensor] - ) -> Optional[torch.Tensor]: - """This batch's sliding-window write loc, or None when there is no loc - this forward or the pool has no sliding-window id space.""" - if out_cache_loc is None or self._swa_write_loc_from_full is None: - return None - return self._swa_write_loc_from_full(out_cache_loc) - - def _swa_write_loc_unified(self, full_loc: torch.Tensor) -> torch.Tensor: - """Sliding-window write loc, derived pointwise from FULL-side physical - ids (phase 2 of the write contract). - """ - ps = self.page_size - offset = full_loc % ps # == virtual_token % page_size - # An unmapped physical page reads back as -1; clamp it rather than let - # the gather wrap onto the v2p table's last element. - virt_page = self._full_p2v_table[full_loc // ps].clamp_(min=0) - return (self._swa_v2p_table[virt_page] * ps + offset).clamp_(min=0) - - # -- token-level translate surface (the mixin / local-attn consumers) ------ + # -- DCP read production --------------------------------------------------- @property def needs_read_translate(self) -> bool: @@ -575,14 +604,6 @@ class KVIndexTranslator: dcp_size = get_parallel().attn_dcp_size if dcp_size > 1: widened_ids = widened_ids // dcp_size - return self.translate_full_attn_ids(widened_ids) - - def translate_full_attn_ids( - self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - """Virtual token ids -> physical full-attention ids (the identity - when no translation is needed, so callers never branch).""" if not self.is_translating: - assert out is None, "passthrough translate takes no out=" - return kv_indices - return self._translate_full(kv_indices, out=out) + return widened_ids + return self._translate_full(widened_ids) diff --git a/python/sglang/srt/mem_cache/kv_loc_plan.py b/python/sglang/srt/mem_cache/kv_loc_plan.py new file mode 100644 index 0000000000..91da2707c6 --- /dev/null +++ b/python/sglang/srt/mem_cache/kv_loc_plan.py @@ -0,0 +1,348 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""One scheduler iteration's KV slot ids, per sub-pool. + +Under the unified pool a KV slot has a virtual id (what the scheduler allocates +and `req_to_token` stores) and, in each sub-pool, a physical id (what that +sub-pool's kernels index). Every forward of one iteration -- the target's, and +a fused draft's, which lives in the target's pages -- writes the same slots and +reads the same rows, and compaction does not move a page while the iteration's +forwards are in flight. So: + + * The write window is translated once per sub-pool, by the plan: the + full-attention ids when the plan is built, another sub-pool's on first + use (`write_ids`). Every writer binds them. + * A read is translated exactly once, only by the translator's read + primitives (`KVIndexTranslator`); a consumer never translates. + * The read table (`read_table`) exists only once a table reader asks for + it, built into that reader's own capture-stable table when it is the + first. Later readers take it; earlier ones translate in their own gather. + * A sub-pool is named by its `IdSpace`; one that indexes virtual ids as they + are (a static pool's, a private draft pool's) translates nothing. + +One plan per iteration: `ForwardBatch.init_new` builds it for a lone forward, a +speculative worker as soon as the iteration's slots are allocated (shared by +its draft and target forwards and direct KV writers), and a graph runner for a +batch over its own buffers (`write_slots`). +""" + +from __future__ import annotations + +import enum +from typing import TYPE_CHECKING, Callable, Dict, Hashable, Optional, Union + +import msgspec +import torch + +if TYPE_CHECKING: + from sglang.srt.mem_cache.kv_index_translator import ( + KVIndexTable, + KVIndexTranslator, + ) + + +class IdSpaceKind(enum.Enum): + """A sub-pool of a runner's KV pool, as the consumers that index it name + it.""" + + # Full-attention layers: the pool's own token ids. + FULL = enum.auto() + # The sliding-window layers of a hybrid pool: the same token ids through + # the sliding-window sub-pool's table. + SLIDING_WINDOW = enum.auto() + + +class IdSpace(msgspec.Struct, frozen=True): + """How one sub-pool names the iteration's slots. + + ``write`` maps the iteration's virtual token ids to this sub-pool's ids + (None: they index it as they are). ``read_v2p`` is the page table its reads + go through (None: reads stay in `req_to_token`, for the consumer to + finish). Runners whose pools share a sub-pool share its ``key``, and with + it a plan's ids for it.""" + + key: Hashable + write: Optional[Callable[[torch.Tensor], torch.Tensor]] = None + read_v2p: Optional[torch.Tensor] = None + + +def window_read_extent(forward_mode, spec_info, write_ids, batch_size: int) -> int: + """How far past ``seq_lens`` a forward reads: the window it writes, when + its ``seq_lens`` do not count that window yet. A verify reads at most + ``draft_token_num`` past them (a ragged verify writes fewer for some rows); + a draft decode reads back its own earlier steps; a draft extend's lengths + take its window only after its batch is built. Zero otherwise.""" + if spec_info is None or write_ids is None or not batch_size: + return 0 + if forward_mode.is_target_verify(): + return spec_info.draft_token_num + if forward_mode.is_decode() or forward_mode.is_draft_extend_v2(): + return write_ids.numel() // batch_size + return 0 + + +class TokenSpan(msgspec.Struct, frozen=True): + """A contiguous run ``[start, stop)`` of a write window's flat tokens (one + half of a two-batch-overlap split), which a reader takes as a view; a + ``slice`` in `Cols` names columns of every row instead.""" + + start: int + stop: int + + +# Part of a `[batch, window]` write window: a slice of its columns, a +# contiguous run of its flat tokens, or a flat index into it (-1: the sink). +Cols = Union[slice, TokenSpan, torch.Tensor] + + +def pad_with_sink(ids: Optional[torch.Tensor], n: int) -> Optional[torch.Tensor]: + """``ids`` padded to ``n`` entries with the sink (id 0 in every space), for + a batch whose write ids cover more lanes than its plan's (a padded batch, + a captured graph's buffer).""" + if ids is None or ids.shape[0] >= n: + return ids + padded = ids.new_zeros(n) + padded[: ids.shape[0]].copy_(ids) + return padded + + +class KVLocPlan: + """The ids of one iteration, in every sub-pool, each computed once.""" + + def __init__( + self, + *, + source: KVIndexTranslator, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + write_virtual: Optional[torch.Tensor], + read_extent: int = 0, + write_slots: Optional[torch.Tensor] = None, + ): + self._source = source + self.req_pool_indices = req_pool_indices + # Committed lengths when the plan is built; every read this iteration + # stays within `seq_lens + read_extent` (the window it writes). + self.seq_lens = seq_lens + self.seq_lens_cpu = seq_lens_cpu + self.read_extent = read_extent + full = source.space(IdSpaceKind.FULL) + if write_slots is not None: + # A runner's own write buffer (graph capture, warmup): already in + # its pool's ids, naming the sink until a replay fills it, and kept + # by address by a captured graph. + assert write_virtual is None + self.write_virtual = None if full.write is not None else write_slots + self.write_physical = write_slots + else: + # Aliases the ScheduleBatch's tensor, which stays virtual for the + # radix tree, the accept path and lazy compaction's in-flight write + # set. + self.write_virtual = write_virtual + # FIXME: a speculative iteration builds its plan before its draft + # runs, so this launch sits on the host path ahead of the draft's + # graph, idle GPU time at low concurrency. Translate the window + # inside the iteration's first captured forward instead, into a + # capture-stable buffer the plan owns, which every later forward + # and direct writer reads -- once per sub-pool, the sliding-window + # ids (derived in `write_ids`) included. + self.write_physical = ( + full.write(write_virtual) + if full.write is not None and write_virtual is not None + else write_virtual + ) + self._full_key = full.key + self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = { + full.key: self.write_physical + } + self._read_tables: Dict[Hashable, KVIndexTable] = {} + + # -- writes ---------------------------------------------------------------- + + def bind(self, batch, reader: KVIndexTranslator, *, cols: Optional[Cols] = None): + """Give ``batch`` (a ForwardBatch or a stand-in) this plan, the part of + the window it writes (``cols``), and its full-attention write ids in + `reader`'s pool. The one way a forward gets its write ids; another + sub-pool's come from `KVIndexTranslator.write_ids`.""" + batch.kv_loc_plan = self + batch.kv_loc_cols = cols + batch.out_cache_loc = self.write_ids(reader, cols=cols) + batch.out_cache_loc_virtual = ( + self.virtual_write_ids(cols=cols) + if self.is_translated_for(reader) + else None + ) + batch.out_cache_loc_is_physical = batch.out_cache_loc is not None + + def bind_replay(self, batch, reader: KVIndexTranslator, *, slots: torch.Tensor): + """`bind`, for the batch a captured graph replays this iteration's + forward with. Its write ids are ``slots``, the runner's capture-stable + buffer that already holds this plan's write ids and names the sink past + them. The virtual mirror stays with the live batch.""" + self.bind(batch, reader) + batch.out_cache_loc = slots + batch.out_cache_loc_virtual = None + + def write_ids( + self, + reader: KVIndexTranslator, + *, + kind: IdSpaceKind = IdSpaceKind.FULL, + cols: Optional[Cols] = None, + ) -> Optional[torch.Tensor]: + """The write window as `reader`'s ``kind`` sub-pool indexes it, or None + when its pool has no such sub-pool. ``cols`` selects part of a + ``[batch, window]`` window, for a forward that writes only part of it: + a slice of columns (a draft step ahead of the verify window it opens), + or a flat index into the window, -1 naming the sink (a ragged verify's + packed rows).""" + space = reader.space(kind) + if space is None: + return None + if space.key not in self._write_ids: + self._write_ids[space.key] = self._window_in(space) + return self._cols(self._write_ids[space.key], cols) + + def _window_in(self, space: IdSpace) -> Optional[torch.Tensor]: + if self.write_virtual is None: + # No window this iteration, or a runner's own slots, which name + # the sink in every other sub-pool too. + if self.write_physical is None: + return None + return torch.zeros_like(self.write_physical) + if space.write is None: + return self.write_virtual + return space.write(self.write_virtual) + + def virtual_write_ids(self, *, cols: Optional[Cols] = None): + """The same columns in the virtual space: the ids the scheduler's + bookkeeping (lazy compaction's in-flight write set, the radix tree) + names.""" + return self._cols(self.write_virtual, cols) + + def _cols(self, ids: Optional[torch.Tensor], cols: Optional[Cols]): + if ids is None or cols is None: + return ids + if isinstance(cols, TokenSpan): + return ids[cols.start : cols.stop] + if isinstance(cols, torch.Tensor): + return torch.where(cols >= 0, ids[cols.clamp(min=0)], 0) + bs = int(self.req_pool_indices.numel()) + if bs == 1: + return ids[cols] + return ids.view(bs, -1)[:, cols].reshape(-1) + + def cols_slice(self, cols: Optional[Cols], tokens: slice) -> Cols: + """The ``tokens`` of a forward that writes ``cols``: the columns of + one half of a batch split in two (two-batch overlap). A contiguous + half stays a `TokenSpan`; only columns spread over the rows become a + flat index into the window. Tokens past the window are left out; the + consumer pads them with the sink.""" + if isinstance(cols, torch.Tensor): + return cols[tokens] + window = ( + self.write_virtual + if self.write_virtual is not None + else self.write_physical + ) + if cols is None or isinstance(cols, TokenSpan): + base = 0 if cols is None else cols.start + end = window.numel() if cols is None else cols.stop + start, stop, step = tokens.indices(end - base) + if step == 1: + return TokenSpan(base + start, base + max(start, stop)) + index = torch.arange(window.numel(), device=window.device) + return self._cols(index, cols)[tokens] + + def reads_from( + self, + source: KVIndexTranslator, + *, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + read_extent: int, + ) -> KVLocPlan: + """This plan's write ids, with reads planned over ``source``'s own + `req_to_token` rows and lengths -- a draft that reads a compact table + of its own while writing the iteration's window. The write ids are + shared, not translated again.""" + plan = KVLocPlan( + source=source, + req_pool_indices=self.req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=None, + read_extent=read_extent, + ) + plan.write_virtual = self.write_virtual + plan.write_physical = self.write_physical + plan._write_ids = self._write_ids + return plan + + # -- reads ----------------------------------------------------------------- + + def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool: + """Whether a reader has had the ``kind`` table built yet.""" + return self._source.space(kind).key in self._read_tables + + def read_table( + self, + *, + kind: IdSpaceKind = IdSpaceKind.FULL, + rows: Optional[int] = None, + into: Optional[torch.Tensor] = None, + ) -> KVIndexTable: + """The ``kind`` page table over ``[0, seq_lens + read_extent)``, built + when a table reader first asks (in place in ``into``, that reader's + capture-stable table, when given) and shared by every later reader. + ``rows=n`` returns exactly n rows; lanes past the plan's batch read the + sink, appended by copying, never by translating again. Where reads stay + virtual (a static pool, or DCP) it is the `req_to_token` passthrough.""" + space = self._source.space(kind) + table = self._read_tables.get(space.key) + if table is None or ( + table.is_translated and rows is not None and table.ids.shape[0] < rows + ): + table = self._source.build_iteration_table( + self, space, rows=rows, previous=table, into=into + ) + self._read_tables[space.key] = table + else: + assert into is None, "a destination goes to the table's first build" + if rows is not None and table.ids.shape[0] > rows: + # Built wider, for a captured graph's padded lanes: this reader's + # kernels size their batch by the table's rows. + table = msgspec.structs.replace( + table, ids=table.ids[:rows], row_ids=table.row_ids[:rows] + ) + return table + + def is_read_by(self, reader: KVIndexTranslator) -> bool: + """Whether `reader` reads this plan's tables: it indexes the plan's + physical ids and gathers the same `req_to_token` rows. A reader with a + table of its own (a draft's compact `req_to_token`) plans its own + reads.""" + return ( + self.is_translated_for(reader) + and reader.req_to_token is self._source.req_to_token + ) + + def is_translated_for(self, reader: KVIndexTranslator) -> bool: + """Whether `reader`'s pool indexes this plan's physical ids: the + target's, and a fused draft's, which lives in the target's full + sub-pool. A pass-through reader (a static pool, a private draft pool) + indexes the virtual ids.""" + full = reader.space(IdSpaceKind.FULL) + return full.write is not None and full.key == self._full_key diff --git a/python/sglang/srt/mem_cache/layout/fused_draft.py b/python/sglang/srt/mem_cache/layout/fused_draft.py index dbe1748af3..c404b4e100 100644 --- a/python/sglang/srt/mem_cache/layout/fused_draft.py +++ b/python/sglang/srt/mem_cache/layout/fused_draft.py @@ -5,45 +5,23 @@ The nesting, outermost first: sub-pool a named pool inside the unified pool, with its own grow direction and frontier. HOST is the role it plays when its entries also carry draft bytes; only "full" hosts a draft here. - slot one token entry of that sub-pool. The draft's KV becomes extra - parts of every slot, after the host's own parts at - `draft_offset_in_entry()`, so ONE slot id per token covers target - and draft: one allocation, one free, one whole-page relocation - carry both. + slot one token entry of that sub-pool. The draft's KV is extra parts of + every slot, after the host's own parts, so ONE slot id per token + covers target and draft: one allocation, one free, one whole-page + relocation carry both. region the draft geometry inside the host's entries. A region is not a `SubPoolSpec`: no grow direction, no frontier logic, only geometry. lane one position in the region's layer dimension -- one draft layer of one runner, present in every slot. -Runners are the draft's execution copies, and the two head shapes place them -oppositely. A replicated head gives EVERY runner all the draft's layers, so -each needs its own lanes or the runners clobber each other's KV (3 runners x -2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token, -depth 1 the one after, and so on) gives each runner ONE depth (8 layers -across 2 runners = 2 lanes). Lane count is therefore not layer count, in -either direction. Each runner owns a contiguous run of lanes, and the runs -tile the region in runner order. +A lane is not a layer. Runners are the draft's execution copies: a +replicated head gives EVERY runner all the draft's layers, on lanes of its own +so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes), +while a per-depth head (one block per predicted position) gives each runner +ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a +contiguous run of lanes, and the runs tile the region in runner order. -Three phases, three moments in boot, and what carries each: - - PLAN on the TARGET, before any memory exists - KVCacheConfigurator._fused_draft_decision() - draft_kv_profile(draft_model_config) what the checkpoint asks for - place_fused_draft(profile, runners) admit, or decline with a reason - KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve - - CARVE as the byte buffer is cut - KVCacheConfigurator._fused_draft_for_pool_factory() resolve once - init_unified_*_pools(fused_draft=...) - MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes() - UnifiedKVPool.build_dense_draft_views() - - BIND on each DRAFT runner, built after the target - KVCacheConfigurator._fused_draft_from_target_buffer(alloc) - bind_fused_draft(...) -> UnifiedDraftKVPool - KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate - -A draft whose layers need another sub-pool kind declines to a private pool. +A draft whose layers need another sub-pool kind declines fusion as a whole. """ from typing import List, Optional, Tuple @@ -57,8 +35,8 @@ from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True): """Geometry of the DRAFT model's K/V rows fused into every slot of a host sub-pool. The draft is a separate checkpoint, so its head geometry and - layer count differ from the host's; its K and V rows are two more parts of - the host entry, indexed by the host's physical token id.""" + layer count can differ from the host's; its K and V rows are two more parts + of the host entry, indexed by the host's physical token id.""" lane_num: int head_num: int @@ -174,7 +152,6 @@ def draft_state_layer_num(draft_model_config) -> int: mambaish = mambaish_config(draft_model_config) if mambaish is None: return 0 - # Optional HF attribute: only conv-chain MTP configs declare it. if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None: return 0 return len(mambaish.mamba2_cache_params.layers) @@ -217,8 +194,8 @@ def draft_kv_profile( class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True): - """`place_fused_draft`'s answer: the placement, or why the draft keeps a - private pool. Neither means fusion simply does not apply.""" + """`place_fused_draft`'s answer: the placement, or why the draft cannot + fuse. Neither means fusion does not apply.""" placement: Optional[FusedDraftPlacement] = None declined: Optional[str] = None @@ -253,8 +230,8 @@ def place_fused_draft( ) -> FusedDraftDecision: """Assign every draft layer of every runner to the host sub-pool whose lifetime covers what the layer reads: a full-attention layer rides in - ``"full"``. A layer kind no host arm serves declines the whole draft to - its private pool.""" + ``"full"``. A layer kind no host sub-pool serves declines the whole + draft.""" counts, reason = _runner_layer_counts(profile, num_runners) if counts is None: return FusedDraftDecision(declined=reason) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index c5ee284822..c6be87a26a 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1871,26 +1871,21 @@ class KVWriteLoc: All location info lives here (in the attention metadata), NOT in the pool: - ``loc``: the generic per-token write location (``out_cache_loc``). - PHYSICAL on every pool: by allocation on non-unified pools, - rebound at ForwardBatch construction (``rebind_write_loc``) on - the unified pool. + PHYSICAL on every pool: by allocation on non-unified pools, the + iteration's plan's write ids on the unified pool (``KVLocPlan.bind``). - ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None`` - otherwise); under the unified pool the translator derives it from the - same rebound loc (``sliding_window_write_loc_for``). - - ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the - construction-time rebind it is the SAME id space as ``loc``, so pools - fall back to ``loc`` when it is ``None`` -- only triton's captured path - still passes its capture-stable - ``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a - same-space alias slated for collapse). + otherwise): the same plan's write ids in the sliding-window sub-pool + (``KVIndexTranslator.write_ids``). + - ``full_loc``: OPTIONAL full-attention-sub-pool location, in the same id + space as ``loc``; pools fall back to ``loc`` when it is ``None``. ``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``); ``loc`` is the generic fallback. Bundling them lets a backend issue one ``set_kv_buffer`` call regardless of pool type. - ``physical`` marks the locs as physical token ids: the batch's write loc - after ``rebind_write_loc``, or ids a backend translated itself. A unified + ``physical`` marks the locs as physical token ids: the write ids a plan + bound to the batch, or ids produced separately and marked so. A unified pool's write door refuses a loc not marked physical; other pools do not check. """ @@ -1910,9 +1905,9 @@ class KVWriteLoc: ) -> KVWriteLoc: """The batch's ``out_cache_loc`` as a write loc, carrying the batch's physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under - the same mark, so it must be derived from that rebound loc (as - ``sliding_window_write_loc_for`` does); a loc produced separately - states its own mark with ``KVWriteLoc(loc, physical=...)``.""" + the same mark, so it must come from the same plan (as + ``KVIndexTranslator.write_ids`` does); a loc produced separately states + its own mark with ``KVWriteLoc(loc, physical=...)``.""" return cls( forward_batch.out_cache_loc, swa_loc, @@ -2095,9 +2090,9 @@ class KVCache(abc.ABC): if self.requires_physical_write_loc and not write_loc_is_physical(loc_info): raise ValueError( f"{where}: write loc is not marked physical. Hand the pool " - "KVWriteLoc.for_batch(forward_batch) after " - "KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, " - "physical=True) for ids translated separately." + "KVWriteLoc.for_batch(forward_batch) for a batch bound to a " + "plan (KVLocPlan.bind), or KVWriteLoc(loc, physical=True) for " + "ids translated separately." ) @abc.abstractmethod @@ -4734,7 +4729,7 @@ class MLATokenToKVPool(KVCache): # Has the WRITE loc arriving here already had the DCP owner rule resolved? # False: this pool takes a WIDENED loc. The unified pool resolves it in - # `KVIndexTranslator.rebind_write_loc` and flips this. + # the plan's write translation (`translate_write_loc`) and flips this. write_loc_is_dcp_resolved = False @property diff --git a/python/sglang/srt/mem_cache/unified_draft_pool.py b/python/sglang/srt/mem_cache/unified_draft_pool.py index ab6798bdca..313a4c38d5 100644 --- a/python/sglang/srt/mem_cache/unified_draft_pool.py +++ b/python/sglang/srt/mem_cache/unified_draft_pool.py @@ -15,15 +15,11 @@ class UnifiedDraftKVPool(MHATokenToKVPool): """Dense draft KV over the draft parts of one host sub-pool's entries. Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside - every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`); - ``layer_lanes`` maps each of this runner's layer ids to its region lane. - Locs arriving through the KVCache API are the target's PHYSICAL token ids, - produced by the allocator's translate (the id-space choke point binds it, - see KVIndexTranslator); the pool exposes `host_allocator` for that - binding. Relocation needs no method here: compaction moves whole page - envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache` - raises so a stray per-slot move fails loudly instead of corrupting the - fused layout. + every slot of the host sub-pool; ``layer_lanes`` maps each of this + runner's layer ids to its region lane. Locs are the target's PHYSICAL + token ids, translated through `host_allocator`. Compaction moves whole + page envelopes on the HOST pool, draft bytes included, so `move_kv_cache` + raises rather than let a per-slot move corrupt the fused layout. """ requires_physical_write_loc = True diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 56931315d4..4039c4b341 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -149,8 +149,7 @@ class MHASubPoolSpec(SubPoolSpec): [ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ] The slot stride stays the one entry, so host and draft views are indexed - by the same physical token id; only their offsets differ. `draft_region - is None` keeps the layout byte-identical to the unfused one. + by the same physical token id; only their offsets differ. """ head_num: int @@ -930,8 +929,9 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool): # Lifetime owned by UnifiedKVPool; do not delete the views. pass - # `rebind_write_loc` already collapsed the widened id and sent the rows this - # rank does not own to the padding sink. + # The plan's write translation (`translate_write_loc`) already collapsed + # the widened id and sent the rows this rank does not own to the padding + # sink. write_loc_is_dcp_resolved = True def get_kv_size_bytes(self): diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index 073ff30891..c5f15baaa8 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -814,7 +814,6 @@ class CPUGraphRunner: forward_batch = ForwardBatch( forward_mode=self.capture_forward_mode, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=input_ids, req_pool_indices=req_pool_indices, @@ -832,6 +831,7 @@ class CPUGraphRunner: num_token_non_padded=self.num_token_non_padded, global_forward_mode=self.capture_forward_mode, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) # Wrap all forward calls with capture_with_skip_cross_attention so that # mllama (and any other encoder-decoder model) sees the correct compile- # time constant for skip_cross_attention during tracing. diff --git a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py index 122c216b6f..b9c23152dd 100644 --- a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py +++ b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py @@ -83,6 +83,7 @@ class ForwardBatchDeepSeekMHAMixin: ): self.prefix_chunk_kv_indices = [] req_to_token = get_req_to_token_pool().req_to_token + translator = get_attn_backend().kv_index_translator for idx in range(self.num_prefix_chunks): chunk_starts = self.prefix_chunk_starts[idx] chunk_seq_lens = self.prefix_chunk_seq_lens[idx] @@ -93,6 +94,18 @@ class ForwardBatchDeepSeekMHAMixin: num_chunk_tokens, dtype=torch.int32, device=device ) + if translator.reads_are_translated: + # The plan's table already holds the chunk's physical ids. + translator.pack_read_stream( + self.kv_loc_plan, + req_pool_indices=self.req_pool_indices, + seq_lens=chunk_seq_lens, + indptr=chunk_cu_seq_lens, + out=chunk_kv_indices, + kv_start_idx=chunk_starts, + ) + self.prefix_chunk_kv_indices.append(chunk_kv_indices) + continue create_chunked_prefix_cache_kv_indices[(self.batch_size,)]( req_to_token, self.req_pool_indices, @@ -107,7 +120,6 @@ class ForwardBatchDeepSeekMHAMixin: self.prefix_chunk_starts_cpu[idx], self.prefix_chunk_seq_lens_cpu[idx], ) - translator = get_attn_backend().kv_index_translator chunk_kv_indices = translator.translate_dcp_read_ids(chunk_kv_indices) self.prefix_chunk_kv_indices.append(chunk_kv_indices) @@ -266,19 +278,28 @@ class ForwardBatchDeepSeekMHAMixin: device=self.req_pool_indices.device, ) kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0) - req_to_token = get_req_to_token_pool().req_to_token - create_flashinfer_kv_indices_triton[(self.batch_size,)]( - req_to_token, - self.req_pool_indices, - self.seq_lens, - kv_indptr, - None, - kv_indices, - req_to_token.shape[1], - ) - # None on a backend that never bound a translator. + # None on a backend that never bound a translator, which reads virtual + # ids; otherwise one gather from the iteration's plan (DCP takes its + # own branch upstream). src = get_attn_backend().kv_index_translator - if src is not None: - kv_indices = src.translate_full_attn_ids(kv_indices) + if src is None: + req_to_token = get_req_to_token_pool().req_to_token + create_flashinfer_kv_indices_triton[(self.batch_size,)]( + req_to_token, + self.req_pool_indices, + self.seq_lens, + kv_indptr, + None, + kv_indices, + req_to_token.shape[1], + ) + else: + src.pack_read_stream( + self.kv_loc_plan, + req_pool_indices=self.req_pool_indices, + seq_lens=self.seq_lens, + indptr=kv_indptr, + out=kv_indices, + ) self.mha_one_shot_kv_indices = kv_indices return kv_indices diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index fd783f7dc5..a7cbd93559 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -43,7 +43,6 @@ from sglang.srt.environ import envs from sglang.srt.kv_canary.req_to_expected_token_ids_manager import ( compute_req_all_ids_info, ) -from sglang.srt.layers.dcp.layout import localize_dcp_indices from sglang.srt.layers.dp_attention import ( DpPaddingMode, dp_slot_in, @@ -77,6 +76,7 @@ if TYPE_CHECKING: from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch + from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm @@ -192,21 +192,6 @@ def _elastic_should_preserve_local_token_counts( return uneven_token_count -def _localize_npu_dcp_out_cache_loc( - out_cache_loc: torch.Tensor, - *, - interleave_size: int, -) -> torch.Tensor: - """Map allocator-global NPU DCP slots to this target rank.""" - parallel = get_parallel() - return localize_dcp_indices( - out_cache_loc, - parallel.dcp_size, - parallel.dcp_rank, - interleave_size, - ) - - class ForwardMode(IntEnum): # Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt). # It is also called "prefill" in common terminology. @@ -532,14 +517,23 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # The original sequence length without being chunked. Qwen-1M related. orig_seq_lens: Optional[torch.Tensor] = None - # The write loc before `rebind_write_loc` replaced it with physical - # ids; a backend re-derives from it into its capture-stable buffer. + # This iteration's KV ids, translated once per sub-pool (`KVLocPlan`). + # Every forward of the iteration holds the same plan; `out_cache_loc` is + # its write window in the full-attention ids this runner's pool indexes, + # and a backend that writes another sub-pool asks the plan for its ids + # (`KVIndexTranslator.write_ids`). + kv_loc_plan: Optional[KVLocPlan] = None + # The part of the plan's window this forward writes (None: all of it). + kv_loc_cols: Optional[Cols] = None + # The same write ids in the virtual space when `out_cache_loc` holds + # translated ones (None on a pool that indexes virtual ids), for the + # scheduler's bookkeeping: lazy compaction's in-flight write set, TBO's + # split, state capture. out_cache_loc_virtual: Optional[torch.Tensor] = None # DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator, # consumed by the Ascend backend for PA_ND block tables. None elsewhere. out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None - # Whether `out_cache_loc` holds physical ids: set by - # KVIndexTranslator.rebind_write_loc; capture-time batches declare it. + # Whether `out_cache_loc` holds physical ids: set by `KVLocPlan.bind`. out_cache_loc_is_physical: bool = False # The indices to track mamba state with mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64 @@ -937,9 +931,16 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): return_hidden_states_before_norm: bool, extend_position_info=None, spec_mrope_positions: Optional[torch.Tensor] = None, + kv_loc_plan: Optional[KVLocPlan] = None, + write_cols: Optional[Cols] = None, ): # init_new must not mutate the input ScheduleBatch; per-forward # overrides go through explicit keyword arguments. + # + # `kv_loc_plan`: the iteration's plan when this forward is one of + # several over the same slots (a speculative iteration), `write_cols` + # the columns of its window this forward writes. Without one, this + # forward is its own iteration and builds the plan itself. # capture_hidden_mode=None means no override: capture the server's # configured maximum so lower-mode requests can share one graph. @@ -1051,15 +1052,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): spec_info=batch.spec_info, ) - # ScheduleBatch and req_to_token keep allocator-global slot identities. - # Preserve that view before exposing rank-local NPU DCP write slots. + # ScheduleBatch and req_to_token keep allocator-global slot identities, + # and so does the plan's window. An NPU DCP target writes rank-local + # slots, which its translator maps the window to at `bind` below; keep + # the global view for the consumers that read it. if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker: ret.origin_out_cache_loc = ret.out_cache_loc - if ret.out_cache_loc is not None: - ret.out_cache_loc = _localize_npu_dcp_out_cache_loc( - ret.out_cache_loc, - interleave_size=model_runner.page_size, - ) ret._maybe_init_non_generation_fields(batch) device = model_runner.device @@ -1073,7 +1071,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): device, ) - model_runner.kv_index_translator.rebind_write_loc(ret) + translator = model_runner.kv_index_translator + if kv_loc_plan is None: + kv_loc_plan = translator.own_plan(ret) + kv_loc_plan.bind(ret, translator, cols=write_cols) if envs.SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.get(): hashed = _hash_rids_to_tensor( @@ -2143,6 +2144,8 @@ def build_inner_fb_view( out_cache_loc=getattr(forward_batch, "out_cache_loc", None), # A caller may hand in another view that does not carry this field. out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None), + kv_loc_plan=getattr(forward_batch, "kv_loc_plan", None), + kv_loc_cols=getattr(forward_batch, "kv_loc_cols", None), origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None), out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None), spec_info=forward_batch.spec_info, diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 4ec4a9e8ca..04f8524d00 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -898,6 +898,7 @@ class ModelRunner: token_to_kv_pool=self.token_to_kv_pool, page_size=self.page_size or 1, device=self.device, + is_draft_worker=self.is_draft_worker, ) def max_shared_logits_buffer_rows(self) -> int: diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index c9551cd281..e0fcc11148 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -662,7 +662,6 @@ class BaseRunner(ABC): forward_batch = ForwardBatch( forward_mode=capture_forward_mode, - out_cache_loc_is_physical=True, batch_size=batch_size, input_ids=input_ids, req_pool_indices=req_pool_indices, @@ -698,6 +697,7 @@ class BaseRunner(ABC): global_forward_mode=capture_forward_mode, lora_ids=lora_ids, ) + mr.kv_index_translator.bind_runner_slots(forward_batch) if buffers.ngram_embedding_info is not None: forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice( diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 67cb669058..c901f3d0c9 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -189,6 +189,8 @@ def build_replay_fb_view( encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None, out_cache_loc=getattr(forward_batch, "out_cache_loc", None), out_cache_loc_virtual=forward_batch.out_cache_loc_virtual, + kv_loc_plan=forward_batch.kv_loc_plan, + kv_loc_cols=forward_batch.kv_loc_cols, origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None), out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None), max_seq_len_override=forward_batch.max_seq_len_override, @@ -1005,7 +1007,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): forward_batch = ForwardBatch( forward_mode=self.capture_forward_mode, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=input_ids, req_pool_indices=req_pool_indices, @@ -1043,6 +1044,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): rids_int=rids_int, bootstrap_room_ids_int=bootstrap_room_ids_int, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) # Trip the coordinator so the hisparse code path is captured into the # graph; backends read it from self.model_runner.hisparse_coordinator. @@ -1118,9 +1120,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self._post_process_after_profile(prof) self._profiler = None - # No pool-side pin to clear: the captured full-physical write loc rides the - # backend's `ForwardMetadata.out_cache_loc_full_physical` (-> KVWriteLoc.full_loc). - def _capture_one_stream(self, stream_idx: Optional[int] = None) -> None: avail_mem = get_available_gpu_memory( self.model_runner.device, @@ -1271,9 +1270,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): if (c := self.model_runner.canary_manager) is not None else contextlib.nullcontext() ) - # Full-physical write loc lives in the attention metadata (the backend's - # `out_cache_loc_full_physical` -> KVWriteLoc.full_loc), so the runner - # wires no buffer here. (SWA write loc rides the `swa_out_cache_loc` rail.) with canary_ctx: shape_key = self._make_graph_key( diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index a67cfce00a..552b735283 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1496,7 +1496,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): with torch.device(self.device): forward_batch = ForwardBatch( forward_mode=ForwardMode.EXTEND, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=_slot("input_ids"), input_embeds=( @@ -1562,6 +1561,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): return_pooled_hidden_states=self.capture_return_pooled_hidden_states, max_seq_len_override=self.max_context_size, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) ngram_manager = self.model_runner.ngram_embedding_manager if ngram_manager.enabled: forward_batch.ngram_embedding_info = NgramEmbeddingInfo.create( @@ -1861,7 +1861,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): static_forward_batch = ForwardBatch( forward_mode=pcg_forward_mode, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=input_ids, input_embeds=input_embeds, @@ -1926,6 +1925,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): ), max_seq_len_override=self.max_context_size, ) + forward_batch.kv_loc_plan.bind_replay( + static_forward_batch, + self.model_runner.kv_index_translator, + slots=out_cache_loc, + ) # The n-gram hasher runs outside the graph and reads this at replay. static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info static_forward_batch.engram_history = forward_batch.engram_history diff --git a/python/sglang/srt/models/dspark.py b/python/sglang/srt/models/dspark.py index 3a2e173a18..c3640ea9c6 100644 --- a/python/sglang/srt/models/dspark.py +++ b/python/sglang/srt/models/dspark.py @@ -803,8 +803,8 @@ class DSparkDraftMixin: attn.attn.v_scale, ) else: - # Translated by the injector (physical by allocation on a plain - # pool). + # `cache_loc` comes from `TargetHiddenKvInjector.ids_for`, + # already in the ids the draft pool indexes. pool.set_kv_buffer( attn.attn, KVWriteLoc(cache_loc, physical=True), diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index 0a87b3a788..af80f5f37c 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -481,12 +481,8 @@ class InklingAttention(nn.Module): do_store = do_bf16_store or do_mxfp8_store metadata = get_attn_backend().forward_metadata if do_store and self.is_local: - # SWA sub-pool + the backend's full->SWA translated write location. + # SWA sub-pool + the plan's sliding-window write ids (backend buffer). loc = metadata.swa_out_cache_loc - elif do_store: - loc = getattr(metadata, "out_cache_loc_full_physical", None) - if loc is None: - loc = forward_batch.out_cache_loc else: loc = forward_batch.out_cache_loc es = q.element_size() @@ -584,10 +580,6 @@ class InklingAttention(nn.Module): metadata = get_attn_backend().forward_metadata if do_store and self.is_local: loc = metadata.swa_out_cache_loc - elif do_store: - loc = getattr(metadata, "out_cache_loc_full_physical", None) - if loc is None: - loc = forward_batch.out_cache_loc else: loc = forward_batch.out_cache_loc is_v2 = forward_batch.forward_mode.is_draft_extend_v2() @@ -691,10 +683,6 @@ class InklingAttention(nn.Module): metadata = get_attn_backend().forward_metadata if do_store and self.is_local: loc = metadata.swa_out_cache_loc - elif do_store: - loc = getattr(metadata, "out_cache_loc_full_physical", None) - if loc is None: - loc = forward_batch.out_cache_loc else: loc = forward_batch.out_cache_loc es = q.element_size() diff --git a/python/sglang/srt/speculative/dflash_info.py b/python/sglang/srt/speculative/dflash_info.py index a004445e52..02becd4a22 100644 --- a/python/sglang/srt/speculative/dflash_info.py +++ b/python/sglang/srt/speculative/dflash_info.py @@ -7,6 +7,7 @@ import torch from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import Cols, IdSpaceKind, KVLocPlan from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, ForwardBatch, @@ -51,6 +52,10 @@ class DFlashVerifyInput(SpecInput): live_seq_lens_cpu: Optional[torch.Tensor] = None # Conservative request-lifetime bound for candidate graph dispatch. candidate_max_seq_len_upper_bound: Optional[int] = None + # The iteration's plan when the caller planned the verify window, and the + # part of it this verify writes (`KVLocPlan.write_ids`). + kv_loc_plan: Optional[KVLocPlan] = None + kv_loc_cols: Optional[Cols] = None def __post_init__(self): super().__init__(spec_input_type=SpecInputType.DFLASH_VERIFY) @@ -101,6 +106,8 @@ class DFlashVerifyInput(SpecInput): target_worker.model_runner, capture_hidden_mode=self.capture_hidden_mode, return_hidden_states_before_norm=False, + kv_loc_plan=self.kv_loc_plan, + write_cols=self.kv_loc_cols, ) can_run_cuda_graph = bool( @@ -133,13 +140,14 @@ class DFlashVerifyInput(SpecInput): paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, translator: KVIndexTranslator, + plan: KVLocPlan, kv_start_idx: Optional[torch.Tensor] = None, kv_indices_buf: Optional[torch.Tensor] = None, - sliding_window: bool = False, + kind: IdSpaceKind = IdSpaceKind.FULL, ): - """CSR verify args, gathered straight into the packed stream. The lens - are widened here and handed to the translator, so nothing materializes - a ``[bs, max_pages]`` rectangle to repack from.""" + """CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and + is widened here; the translator packs the read ids straight into + ``kv_indices``.""" device = req_pool_indices.device bs = req_pool_indices.numel() @@ -179,14 +187,14 @@ class DFlashVerifyInput(SpecInput): dtype=torch.int32, device=device, ) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=cum_kv_seq_len, - total_tokens=paged_kernel_lens_sum + kv_indices_extra, out=kv_indices, kv_start_idx=kv_start_idx, - sliding_window=sliding_window, + kind=kind, ) mask = self.custom_mask if mask is not None: diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 773a6cf7af..70dc79a1bf 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -26,6 +26,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.managers.tp_worker import TpModelWorker +from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( @@ -1744,6 +1745,11 @@ class DFlashWorkerV2(BaseSpecWorker): return out_tokens + def _draft_write_ids(self, plan: KVLocPlan) -> torch.Tensor: + """The plan's write ids as the draft pool indexes them: the target's + physical ids for a fused draft, the virtual ids for a private one.""" + return plan.write_ids(self.draft_model_runner.kv_index_translator) + def _append_target_hidden_to_draft_kv_by_loc( self, *, @@ -1845,17 +1851,9 @@ class DFlashWorkerV2(BaseSpecWorker): if commit_lens.dtype != torch.int32: commit_lens = commit_lens.to(torch.int32) - # On a translating pool the translate writes a new tensor, so the - # callers' locs stay virtual: the post-verify 2-D buffer is re-read as - # virtual ids by the compact req_to_token rebuild. On a plain pool it - # returns `cache_loc` itself. - translator = self.draft_model_runner.kv_index_translator - cache_loc = translator.translate_full_attn_ids(cache_loc) - if cache_loc_2d is not None: - # Same ids, so reshape the translated copy rather than translate - # them a second time. - cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape) - + # `cache_loc` arrives in the draft pool's id space (`_draft_write_ids`); + # the callers' window buffers stay virtual for the compact + # req_to_token rebuild. with ( torch.inference_mode(), draft_tp_context(self.draft_owns_attention), @@ -2265,7 +2263,9 @@ class DFlashWorkerV2(BaseSpecWorker): batch, pp_proxy_tensors=pp_proxy_tensors, capture_hidden_mode=CaptureHiddenMode.FULL, + return_kv_loc_plan=True, ) + kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None logits_output, next_token_ids = ( batch_output.logits_output, @@ -2316,9 +2316,10 @@ class DFlashWorkerV2(BaseSpecWorker): ctx_lens, int(sum(batch.extend_lens)), ) + # The draft KV goes to the slots the target prefill just wrote. self._append_target_hidden_to_draft_kv_by_loc( target_hidden=logits_output.hidden_states, - cache_loc=batch.out_cache_loc, + cache_loc=self._draft_write_ids(kv_loc_plan), positions=positions, extend_lens=ctx_lens, ) @@ -2494,6 +2495,19 @@ class DFlashWorkerV2(BaseSpecWorker): positions = positions_2d.reshape(-1) verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1) + # The iteration's window -- the draft block, the verify and the + # target-hidden KV writes all write it -- planned once. + kv_loc_plan = self.target_worker.model_runner.kv_index_translator.plan( + req_pool_indices=batch.req_pool_indices, + seq_lens=prefix_lens, + seq_lens_cpu=( + batch.seq_lens_cpu + if batch.seq_lens_cpu is not None + else draft_input.nxt_kv_lens_cpu + ), + write_virtual=verify_out_cache_loc, + read_extent=block_size, + ) seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs] if self.use_compact_draft_cache: @@ -2555,10 +2569,20 @@ class DFlashWorkerV2(BaseSpecWorker): ), global_num_token_non_padded_cpu=bs * block_size, ) - # Hand-built draft batch bypasses ForwardBatch.init_new: under the - # unified pool the write loc must be rebound to the draft's - # kernel-facing ids here (no-op on plain pools). - self.draft_model_runner.kv_index_translator.rebind_write_loc(forward_batch) + # The draft block writes the whole window. A compact draft reads its + # own `req_to_token` rows over its own lengths. + draft_translator = self.draft_model_runner.kv_index_translator + draft_plan = ( + kv_loc_plan.reads_from( + draft_translator, + seq_lens=draft_seq_lens, + seq_lens_cpu=seq_lens_cpu, + read_extent=block_size, + ) + if self.use_compact_draft_cache + else kv_loc_plan + ) + draft_plan.bind(forward_batch, draft_translator) if self.selector is not None or self.lilicorr is not None: self._selector_sample = None @@ -2713,6 +2737,7 @@ class DFlashWorkerV2(BaseSpecWorker): draft_token_num=int(self.block_size), custom_mask=custom_mask, capture_hidden_mode=CaptureHiddenMode.FULL, + kv_loc_plan=kv_loc_plan, ) batch.out_cache_loc = verify_out_cache_loc @@ -2877,10 +2902,11 @@ class DFlashWorkerV2(BaseSpecWorker): # Consume in this step: every decode graph size shares one aux output, # which the next target forward overwrites (resolve_aux_hidden_states_width). + cache_loc = self._draft_write_ids(kv_loc_plan) self._append_target_hidden_to_draft_kv_by_loc( target_hidden=hidden.reshape(-1, hidden.shape[-1]), - cache_loc=verify_out_cache_loc, - cache_loc_2d=verify_out_cache_loc_2d, + cache_loc=cache_loc, + cache_loc_2d=cache_loc.view(verify_out_cache_loc_2d.shape), positions=positions, commit_lens=commit_lens, ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_draft.py b/python/sglang/srt/speculative/dspark_components/dspark_draft.py index e71aadf4ab..290d202e15 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_draft.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_draft.py @@ -360,7 +360,7 @@ class DraftBlockProposer: ) # Hand-built batch bypasses ForwardBatch.init_new: rebind the write # loc to the draft's kernel-facing ids (no-op on plain pools). - self.draft_model_runner.kv_index_translator.rebind_write_loc(idle_batch) + self.draft_model_runner.kv_index_translator.bind_own_plan(idle_batch) self._fill_dp_moe_sync_metadata(idle_batch, batch) with torch.inference_mode(): self.draft_model_runner.forward(idle_batch) @@ -435,10 +435,10 @@ class DraftBlockProposer: ), global_num_token_non_padded_cpu=draft_num_tokens, ) - # Hand-built batch bypasses ForwardBatch.init_new: rebind the write - # loc to the draft's kernel-facing ids (no-op on plain pools). - self.draft_model_runner.kv_index_translator.rebind_write_loc( - draft_forward_batch + verify_window.kv_loc_plan.bind( + draft_forward_batch, + self.draft_model_runner.kv_index_translator, + cols=slice(0, query_token_num), ) self._fill_dp_moe_sync_metadata(draft_forward_batch, batch) graph_runner = self.draft_model_runner.decode_cuda_graph_runner diff --git a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py index 00c4fb1f70..6606912e5c 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_kv_inject.py @@ -5,12 +5,12 @@ import torch from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( is_unified_kv_triton, ) -from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( BuildCommitInjectLayout, build_unified_commit_inject_layout, ) from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout @@ -32,6 +32,11 @@ class TargetHiddenKvInjector: self.verify_num_draft_tokens = verify_num_draft_tokens self._block_pos_offsets = block_pos_offsets + def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor: + """The plan's write ids as the draft pool indexes them: the target's + physical ids for a fused draft, the virtual ids for a private one.""" + return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols) + def inject_target_hidden( self, *, @@ -70,16 +75,7 @@ class TargetHiddenKvInjector: device=device, dtype=torch.int64, non_blocking=True ) - # On a translating pool the translate writes a new tensor, so the - # callers' locs stay virtual; on a plain pool it returns `cache_loc` - # itself. - translator = self.draft_model_runner.kv_index_translator - cache_loc = translator.translate_full_attn_ids(cache_loc) - if cache_loc_2d is not None: - # Callers pass `cache_loc` viewed [bs, verify_len]: reshape the - # translated copy rather than translate the same ids twice. - cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape) - + # `cache_loc` arrives in the draft pool's id space (`ids_for`). pool = self.draft_model_runner.token_to_kv_pool if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"): if target_hidden_is_projected: @@ -214,6 +210,7 @@ class TargetHiddenKvInjector: hidden_strided: torch.Tensor, commit_lens: torch.Tensor, bs: int, + kv_loc_plan: KVLocPlan, ) -> None: stride = self.verify_num_draft_tokens prefix_lens = batch.seq_lens @@ -252,15 +249,7 @@ class TargetHiddenKvInjector: return positions_2d = prefix_lens.unsqueeze(1) + self._block_pos_offsets - verify_cache_loc = assign_extend_cache_locs_func( - req_pool_indices=batch.req_pool_indices, - req_to_token=self.model_runner.req_to_token_pool.req_to_token, - start_offset=prefix_lens, - end_offset=prefix_lens + stride, - batch_size=bs, - draft_token_num=stride, - device=self.device, - ) + verify_cache_loc = self.ids_for(kv_loc_plan) verify_cache_loc_2d = verify_cache_loc.view(bs, stride) self.inject_target_hidden( target_hidden=hidden.reshape(-1, hidden.shape[-1]), diff --git a/python/sglang/srt/speculative/dspark_components/dspark_planner.py b/python/sglang/srt/speculative/dspark_components/dspark_planner.py index b025f10009..95e50081d8 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_planner.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_planner.py @@ -20,6 +20,7 @@ from sglang.srt.managers.overlap_utils import ( ResolvedConfidence, ) from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments @@ -65,6 +66,9 @@ class VerifyWindow(msgspec.Struct, frozen=True): positions_2d: torch.Tensor verify_cache_loc: torch.Tensor verify_cache_loc_2d: torch.Tensor + # The iteration's plan of `verify_cache_loc`: the draft block, the verify + # and the target-hidden KV writes all take their ids from it. + kv_loc_plan: KVLocPlan class DSparkVerifyPlanner: @@ -827,7 +831,10 @@ def alloc_verify_window( verify_num_draft_tokens: int, block_pos_offsets: torch.Tensor, model_runner, + seq_lens_cpu: Optional[torch.Tensor], ) -> VerifyWindow: + """``seq_lens_cpu``: a host bound on ``batch.seq_lens``, which sizes the + plan's read table.""" prefix_lens = batch.seq_lens verify_w = verify_num_draft_tokens positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets @@ -845,6 +852,13 @@ def alloc_verify_window( positions_2d=positions_2d, verify_cache_loc=verify_cache_loc, verify_cache_loc_2d=verify_cache_loc_2d, + kv_loc_plan=model_runner.kv_index_translator.plan( + req_pool_indices=batch.req_pool_indices, + seq_lens=prefix_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=verify_cache_loc, + read_extent=verify_w, + ), ) diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index 4a1615c5f0..44439c377b 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -312,6 +312,7 @@ class TargetVerifyExecutor: custom_mask=None, capture_hidden_mode=CaptureHiddenMode.FULL, live_seq_lens_cpu=batch.seq_lens_cpu, + kv_loc_plan=verify_window.kv_loc_plan, ) batch.out_cache_loc = verify_cache_loc seq_lens_cpu_backup = batch.seq_lens_cpu @@ -388,6 +389,7 @@ class TargetVerifyExecutor: hidden_strided=hidden_strided, commit_lens=commit_lens, bs=bs, + kv_loc_plan=verify_window.kv_loc_plan, ) return hidden = logits_output.hidden_states @@ -404,10 +406,11 @@ class TargetVerifyExecutor: state_slot = ( batch.req_pool_indices[:bs].view(-1, 1).expand(bs, vlen).reshape(-1) ) + cache_loc = self.kv_injector.ids_for(verify_window.kv_loc_plan) self.kv_injector.inject_target_hidden( target_hidden=hidden.reshape(-1, hidden.shape[-1]), - cache_loc=verify_window.verify_cache_loc, - cache_loc_2d=verify_window.verify_cache_loc_2d, + cache_loc=cache_loc, + cache_loc_2d=cache_loc.view(verify_window.verify_cache_loc_2d.shape), positions=verify_window.positions_2d.reshape(-1), commit_lens=commit_lens, state_slot=state_slot, @@ -420,6 +423,7 @@ class TargetVerifyExecutor: layout: RaggedVerifyLayout, ragged_window: RaggedVerifyWindow, sampling_info, + kv_loc_plan, ) -> TargetVerifyResult: verify_input = DFlashVerifyInput( draft_token=ragged_window.verify_ids, @@ -429,6 +433,9 @@ class TargetVerifyExecutor: capture_hidden_mode=CaptureHiddenMode.FULL, ragged_verify_layout=layout, live_seq_lens_cpu=batch.seq_lens_cpu, + # The packed rows are a selection of the planned verify window. + kv_loc_plan=kv_loc_plan, + kv_loc_cols=ragged_window.window_index, ) batch.out_cache_loc = ragged_window.verify_cache_loc seq_lens_cpu_backup = batch.seq_lens_cpu @@ -461,6 +468,7 @@ class TargetVerifyExecutor: bs: int, device: str, sampling_info, + verify_window: VerifyWindow, inject_gate: bool = False, ) -> tuple[TargetVerifyResult, torch.Tensor]: ragged_window = BuildRaggedVerifyWindow.execute( @@ -480,6 +488,7 @@ class TargetVerifyExecutor: layout=layout, ragged_window=ragged_window, sampling_info=sampling_info, + kv_loc_plan=verify_window.kv_loc_plan, ) logits_output = target_verify.logits_output diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 490c5798b6..3f24ab7c5b 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -642,6 +642,7 @@ class DSparkWorkerV2(BaseSpecWorker): batch, pp_proxy_tensors=pp_proxy_tensors, capture_hidden_mode=CaptureHiddenMode.FULL, + return_kv_loc_plan=True, ) # BCG replay skips model-side Python, so re-evaluate the same pure predicate. target_hidden_is_projected = ( @@ -705,7 +706,9 @@ class DSparkWorkerV2(BaseSpecWorker): repeats, output_size=num_tokens, ) - cache_loc = batch.out_cache_loc + # The draft KV goes to the slots the target prefill just wrote. + cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan) + batch_output.kv_loc_plan = None token_indices = logits_output.hidden_states_token_indices if token_indices is not None: cache_loc = cache_loc[token_indices] @@ -842,6 +845,11 @@ class DSparkWorkerV2(BaseSpecWorker): verify_num_draft_tokens=self.verify_num_draft_tokens, block_pos_offsets=self._block_pos_offsets, model_runner=self.model_runner, + seq_lens_cpu=( + batch.seq_lens_cpu + if batch.seq_lens_cpu is not None + else draft_input.nxt_kv_lens_cpu + ), ) sampling_info = batch.sampling_info @@ -936,6 +944,7 @@ class DSparkWorkerV2(BaseSpecWorker): bs=bs, device=device, sampling_info=sampling_info, + verify_window=verify_window, inject_gate=fold_eligible, ) else: diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 1896b29750..4a5ec748d4 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -459,7 +459,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): forward_batch = ForwardBatch( forward_mode=ForwardMode.DECODE, - out_cache_loc_is_physical=True, batch_size=num_seqs, input_ids=None, req_pool_indices=req_pool_indices, @@ -485,6 +484,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL ), ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) def run_once(): self.draft_attn_backend.init_forward_metadata_in_graph(forward_batch) diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index fc04c19723..ddc5bd0d95 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -424,7 +424,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): forward_batch = ForwardBatch( forward_mode=self.forward_mode, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=input_ids, req_pool_indices=req_pool_indices, @@ -448,6 +447,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): spec_info=spec_info, capture_hidden_mode=CaptureHiddenMode.LAST, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) if self.buffers.dsa_seed_topk_capture is not None: spec_info.dsa_seed_topk_capture = self.buffers.dsa_seed_topk_capture[ @@ -649,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): encoder_lens=None, out_cache_loc=buffers.out_cache_loc[:num_tokens], out_cache_loc_virtual=forward_batch.out_cache_loc_virtual, + kv_loc_plan=forward_batch.kv_loc_plan, + kv_loc_cols=forward_batch.kv_loc_cols, out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None), # Virtual input stays separate from the backend's physical buffer. mamba_track_indices=track_indices(buffers.mamba_track_indices, bs), diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index 4fd02f6282..0fa2baa73a 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -5,6 +5,7 @@ from typing import Callable, List, Optional import torch from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode from sglang.srt.runtime_context import get_spec from sglang.srt.speculative.spec_info import SpecInput, SpecInputType @@ -34,6 +35,9 @@ class EagleVerifyInput(SpecInput): draft_probs: torch.Tensor = None prepared_out_cache_loc: Optional[torch.Tensor] = None prepared_mrope_positions: Optional[torch.Tensor] = None + # The iteration's plan when the draft planned the verify window + # (`prepared_out_cache_loc`); verify and draft extend take it. + kv_loc_plan: Optional[KVLocPlan] = None # Shape info for padding num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num. @@ -90,11 +94,12 @@ class EagleVerifyInput(SpecInput): paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, translator: KVIndexTranslator, - sliding_window: bool = False, + plan: KVLocPlan, + kind: IdSpaceKind = IdSpaceKind.FULL, ): - """CSR verify args, gathered straight into the packed stream. The lens - are widened here and handed to the translator, so nothing materializes - a ``[bs, max_pages]`` rectangle to repack from.""" + """CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and + is widened here; the translator packs the read ids straight into + ``kv_indices``.""" device = req_pool_indices.device batch_size = req_pool_indices.numel() qo_indptr = torch.arange( @@ -113,13 +118,13 @@ class EagleVerifyInput(SpecInput): total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=cum_kv_seq_len, - total_tokens=total_tokens, out=kv_indices, - sliding_window=sliding_window, + kind=kind, ) mask_numel = ( paged_kernel_lens_sum * self.draft_token_num @@ -402,10 +407,11 @@ class EagleDraftExtendInput(SpecInput): paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: Optional[int], translator: KVIndexTranslator, - sliding_window: bool = False, + plan: KVLocPlan, + kind: IdSpaceKind = IdSpaceKind.FULL, ): - """Draft-extend CSR args. The lens already include the window, so - unlike verify nothing is widened here.""" + """Draft-extend CSR args. ``paged_kernel_lens`` already includes the + draft-extend window.""" device = req_pool_indices.device bs = self.num_correct_drafts.numel() # Constant num_tokens_per_req qo layout (required for cuda-graph capture). @@ -426,12 +432,12 @@ class EagleDraftExtendInput(SpecInput): paged_kernel_lens_sum, dtype=torch.int32, device=device ) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=cum_kv_seq_len, - total_tokens=paged_kernel_lens_sum, out=kv_indices, - sliding_window=sliding_window, + kind=kind, ) return kv_indices, cum_kv_seq_len, qo_indptr, None diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 153ca448ea..c6380e0433 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -607,6 +607,7 @@ def eagle_prepare_for_verify( capture_hidden_mode=capture_mode, return_hidden_states_before_norm=False, spec_mrope_positions=verify_input.prepared_mrope_positions, + kv_loc_plan=verify_input.kv_loc_plan, ) # Run attention backend plan and cuda graph preparation diff --git a/python/sglang/srt/speculative/eagle_worker_common.py b/python/sglang/srt/speculative/eagle_worker_common.py index 7120128866..3b035cf9c9 100644 --- a/python/sglang/srt/speculative/eagle_worker_common.py +++ b/python/sglang/srt/speculative/eagle_worker_common.py @@ -55,6 +55,8 @@ if _is_cpu: if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.tp_worker import TpModelWorker + from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.speculative.eagle_draft_cuda_graph_runner import ( @@ -121,7 +123,10 @@ def prepare_for_draft_extend( return_hidden_states_before_norm: bool, widened_out_cache_loc: Optional[torch.Tensor] = None, widened_positions: Optional[torch.Tensor] = None, + kv_loc_plan: Optional[KVLocPlan] = None, ): + """``kv_loc_plan``: the iteration's plan, whose window the draft extend + writes again (the verify window); a widened window plans its own.""" bs = len(batch.seq_lens) # Optional window widening (num_front_tokens=0 -> off): prepend that many # rows below the boundary. Locs/positions arrive precomputed; token/hidden @@ -204,6 +209,7 @@ def prepare_for_draft_extend( capture_hidden_mode=capture_mode, return_hidden_states_before_norm=return_hidden_states_before_norm, extend_position_info=extend_position_info, + kv_loc_plan=None if widen else kv_loc_plan, ) # Forward sees post-write length (draft extend writes num_draft_tokens # slots); mutation stays on forward_batch to preserve SB.seq_lens. @@ -243,18 +249,35 @@ def prepare_for_draft( draft_model_runner: ModelRunner, topk: int, num_steps: int, -): - + *, + target_translator: Optional[KVIndexTranslator] = None, + num_draft_tokens: Optional[int] = None, +) -> tuple[ForwardBatch, bool, Optional[KVLocPlan]]: + """The draft-decode batch. With ``target_translator``, a chain draft plans + the iteration's whole write window (the ``num_draft_tokens`` slots past + ``seq_lens`` that verify and draft extend also write) and writes its first + ``num_steps`` columns; that plan is returned, else None.""" prepared_positions = None prepared_mrope_positions = None + kv_loc_plan = None if not batch.forward_mode.is_idle(): bs = len(batch.seq_lens) # Assign cache locations (draft-write targets). page_size = batch.token_to_kv_pool_allocator.page_size + plans_window = ( + target_translator is not None + and topk == 1 + and num_draft_tokens is not None + and num_draft_tokens > num_steps + and not _is_cpu + ) + # A chain draft's slots lead the iteration's window: gather the whole + # window in the same launch. + window_cols = num_draft_tokens if plans_window else num_steps if page_size == 1 or topk == 1: batch.out_cache_loc = torch.empty( - (bs * topk * num_steps,), + (bs * topk * window_cols,), dtype=torch.int64, device=batch.device, ) @@ -285,13 +308,21 @@ def prepare_for_draft( batch.out_cache_loc, req_to_token_pool.req_to_token.shape[1], topk, - num_steps, + window_cols, positions=prepared_positions, mrope=prepared_mrope_positions, BS=bs, WRITE_POSITIONS=prepared_positions is not None, WRITE_MROPE=prepared_mrope_positions is not None, ) + if plans_window: + kv_loc_plan = target_translator.plan( + req_pool_indices=batch.req_pool_indices, + seq_lens=batch.seq_lens, + seq_lens_cpu=batch.seq_lens_cpu, + write_virtual=batch.out_cache_loc, + read_extent=num_draft_tokens, + ) else: # page_size > 1 + topk > 1: per-branch page-aligned draft pages. # Reduce out_cache_loc from the page-aligned tree region down to the @@ -352,11 +383,13 @@ def prepare_for_draft( capture_hidden_mode=capture_mode, return_hidden_states_before_norm=False, spec_mrope_positions=prepared_mrope_positions, + kv_loc_plan=kv_loc_plan, + write_cols=None if kv_loc_plan is None else slice(0, num_steps), ) can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph( forward_batch ) - return forward_batch, can_run_decode_cuda_graph + return forward_batch, can_run_decode_cuda_graph, kv_loc_plan def build_eagle_verify_input( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index b05bd5872d..debbfc7bc5 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -41,6 +41,7 @@ from sglang.srt.layers.moe.utils import ( from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.managers.tp_worker import TpModelWorker +from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.cuda_graph_config import ( Backend, Phase, @@ -606,7 +607,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): def draft(self, batch: ScheduleBatch, *, with_topology: bool = False): draft_input: EagleDraftInput = batch.spec_info - forward_batch, can_run_decode_cuda_graph = prepare_for_draft( + forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft( draft_input, self.req_to_token_pool, batch, @@ -614,6 +615,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): self.draft_runner, self.topk, self.speculative_num_steps, + target_translator=self.target_worker.model_runner.kv_index_translator, + num_draft_tokens=self.speculative_num_draft_tokens, ) if ( can_run_decode_cuda_graph @@ -673,6 +676,10 @@ class EagleDraftWorker(EagleDraftWorkerBase): tree_mask_mode=self.tree_mask_mode, device=self.device, ) + if kv_loc_plan is not None: + # Verify and draft extend write the window the draft planned. + verify_input.kv_loc_plan = kv_loc_plan + verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual if with_topology: # PP+spec relays the tree so every stage rebuilds the same verify # input; the mask build needs the topology this one was built from. @@ -900,6 +907,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): target_hidden_states: torch.Tensor, next_token_ids: torch.Tensor, mm_input_embeds: Optional[torch.Tensor] = None, + kv_loc_plan: Optional[KVLocPlan] = None, ): """ Run draft model extend to correctly fill the KV cache. @@ -908,6 +916,8 @@ class EagleDraftWorker(EagleDraftWorkerBase): batch: The batch to run. target_hidden_states: Hidden states from the target model forward next_token_ids: Next token ids generated from the target forward. + kv_loc_plan: The target prefill's plan; the draft writes the same + slots. """ # Construct input_ids if not batch.forward_mode.is_idle(): @@ -959,6 +969,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): self.draft_runner, capture_hidden_mode=capture_hidden_mode, return_hidden_states_before_norm=False, + kv_loc_plan=kv_loc_plan, ) forward_batch.return_logprob = False if mm_input_embeds is not None: @@ -1035,7 +1046,11 @@ class EagleDraftWorker(EagleDraftWorkerBase): return buf[:num_tokens] def _draft_extend_for_decode( - self, batch: ScheduleBatch, batch_result: GenerationBatchResult + self, + batch: ScheduleBatch, + batch_result: GenerationBatchResult, + *, + kv_loc_plan: Optional[KVLocPlan] = None, ): # Cast to int64 before entering plan stream to avoid cross-stream # synchronization issues with .to() inside the plan stream context. @@ -1093,6 +1108,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): self.draft_runner, self.cuda_graph_runner_for_draft_extend, return_hidden_states_before_norm=False, + kv_loc_plan=kv_loc_plan, ) if self.plan_stream: @@ -1412,7 +1428,9 @@ class EAGLEWorkerV2(BaseSpecWorker): speculative_moe_a2a_backend_context(), spec_stage_span("draft_extend"), ): - self.draft_worker._draft_extend_for_decode(batch, batch_output) + self.draft_worker._draft_extend_for_decode( + batch, batch_output, kv_loc_plan=verify_input.kv_loc_plan + ) if ( get_parallel().pp_size > 1 @@ -1468,7 +1486,9 @@ class EAGLEWorkerV2(BaseSpecWorker): batch, pp_proxy_tensors=pp_proxy_tensors, capture_hidden_mode=target_capture_mode, + return_kv_loc_plan=self._draft_worker is not None, ) + kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. # Extend processed L prompt tokens; next verify iter expects same L. @@ -1505,6 +1525,7 @@ class EAGLEWorkerV2(BaseSpecWorker): batch_output.logits_output.hidden_states, batch_output.next_token_ids, batch_output.logits_output.mm_input_embeds, + kv_loc_plan=kv_loc_plan, ) return batch_output diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index 458b1fc013..15733aba40 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -302,7 +302,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner): forward_batch = ForwardBatch( forward_mode=ForwardMode.DECODE, - out_cache_loc_is_physical=True, batch_size=expanded_bs, input_ids=None, req_pool_indices=req_pool_indices, @@ -321,6 +320,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner): spec_info=spec_info, capture_hidden_mode=CaptureHiddenMode.LAST, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) def run_once(): # Record the metadata rebuild against the committed target-prefix diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py index 5a60e258df..b2ab12c00c 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py @@ -365,6 +365,8 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): encoder_lens=None, out_cache_loc=getattr(forward_batch, "out_cache_loc", None), out_cache_loc_virtual=forward_batch.out_cache_loc_virtual, + kv_loc_plan=forward_batch.kv_loc_plan, + kv_loc_cols=forward_batch.kv_loc_cols, spec_info=None, ) with self._frozen_kv_target_view(forward_batch): diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 117307dc0c..6c67b1b471 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -99,6 +99,7 @@ if is_npu(): fill_draft_extend_prepare_buffers = fill_draft_extend_prepare_buffers_native if TYPE_CHECKING: + from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan from sglang.srt.speculative.multi_layer_eagle_worker_v2 import ( MultiLayerEagleDraftWorker, ) @@ -333,7 +334,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): # Forward batch forward_batch = ForwardBatch( forward_mode=self.forward_mode, - out_cache_loc_is_physical=True, batch_size=bs, input_ids=input_ids, req_pool_indices=req_pool_indices, @@ -361,6 +361,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): global_num_token_non_padded_cpu=self.captured_req_width * bs, return_hidden_states_before_norm=True, ) + self.model_runner.kv_index_translator.bind_runner_slots(forward_batch) return forward_batch def _postprocess_forward_batch(self, forward_batch: ForwardBatch, bs: int): @@ -476,6 +477,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): seq_lens_cpu: Optional[torch.Tensor], *, out_cache_loc_virtual: Optional[torch.Tensor], + kv_loc_plan: Optional[KVLocPlan], + kv_loc_cols: Optional[Cols], ): """Init this step's attention metadata for the prepared bucket and replay its graph. Buffers must already be populated by the composite @@ -501,6 +504,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): # Virtual input stays separate from the backend's physical buffer. mamba_track_indices=track_indices(buffers.mamba_track_indices, bs), out_cache_loc_virtual=out_cache_loc_virtual, + kv_loc_plan=kv_loc_plan, + kv_loc_cols=kv_loc_cols, spec_info=spec_info, ) if ( @@ -550,6 +555,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: self.num_front_tokens = 0 self.prune_draft_extend_logits = False self._out_cache_loc_virtual = None + self._kv_loc_plan = None + self._kv_loc_cols = None self._init_and_capture() @@ -740,11 +747,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: req_pool_indices, out_cache_loc, positions=None, - out_cache_loc_virtual=None, + out_cache_loc_virtual, + kv_loc_plan, ): - # Staging runs before `prepare`, so the rail arrives here rather than - # off a ForwardBatch; `prepare` overwrites it with the batch's own. + # Staging runs before `prepare` has a ForwardBatch, so the virtual write + # ids and the plan come in here; `prepare` replaces them with the batch's. self._out_cache_loc_virtual = out_cache_loc_virtual + self._kv_loc_plan = kv_loc_plan + self._kv_loc_cols = None raw_bs = req_pool_indices.shape[0] bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs) buffers = self.buffers @@ -780,6 +790,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width], mamba_track_indices=track_indices(buffers.mamba_track_indices, bs), out_cache_loc_virtual=self._out_cache_loc_virtual, + kv_loc_plan=self._kv_loc_plan, + kv_loc_cols=self._kv_loc_cols, ) for backend in backends: backend.init_forward_metadata_out_graph(batch) @@ -792,6 +804,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: num_tokens = raw_bs * self.captured_req_width self._out_cache_loc_virtual = forward_batch.out_cache_loc_virtual + self._kv_loc_plan = forward_batch.kv_loc_plan + self._kv_loc_cols = forward_batch.kv_loc_cols # Bucketize to a captured batch size (padding the tail). if self.require_mlp_tp_gather: @@ -900,6 +914,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner: self._replay_spec_info, self.seq_lens_cpu, out_cache_loc_virtual=self._out_cache_loc_virtual, + kv_loc_plan=self._kv_loc_plan, + kv_loc_cols=self._kv_loc_cols, ) raw_bs = self.raw_bs raw_num_tokens = self.raw_num_tokens diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index 730830d388..d1b7050720 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -17,7 +17,7 @@ from __future__ import annotations import logging import time from dataclasses import replace -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import torch @@ -107,6 +107,7 @@ _is_cpu = is_cpu() if TYPE_CHECKING: + from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan from sglang.srt.model_executor.model_runner import ModelRunner, ModelRunnerOutput @@ -460,7 +461,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): def draft(self, batch: ScheduleBatch): draft_input: EagleDraftInput = batch.spec_info - forward_batch, can_run_decode_cuda_graph = prepare_for_draft( + forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft( draft_input, self.req_to_token_pool, batch, @@ -468,12 +469,14 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): self.draft_runner_list[0], self.topk, self.speculative_num_steps, + target_translator=self.target_worker.model_runner.kv_index_translator, + num_draft_tokens=self.speculative_num_draft_tokens, ) # Run draft parent_list, top_scores_index, draft_tokens = self.draft_forward(forward_batch) - return build_eagle_verify_input( + verify_input = build_eagle_verify_input( batch, draft_input, parent_list, @@ -487,6 +490,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): tree_mask_mode=self.tree_mask_mode, device=self.device, ) + if kv_loc_plan is not None: + # Verify and draft extend write the window the draft planned. + verify_input.kv_loc_plan = kv_loc_plan + verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual + return verify_input def draft_forward(self, forward_batch: ForwardBatch): # Parse args @@ -596,6 +604,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): batch: ScheduleBatch, target_hidden_states: torch.Tensor, next_token_ids: torch.Tensor, + kv_loc_plan: Optional[KVLocPlan] = None, ): """ Run draft model extend to correctly fill the KV cache. @@ -604,6 +613,8 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): batch: The batch to run. target_hidden_states: Hidden states from the target model forward next_token_ids: Next token ids generated from the target forward. + kv_loc_plan: The target prefill's plan; the draft writes the same + slots. """ # The draft embed clamps unconditionally (to tolerate multimodal pad # sentinels), so probe next_token_ids here first -- otherwise a corrupted id @@ -641,6 +652,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): self.draft_runner_list[0], capture_hidden_mode=draft_capture_hidden_mode, return_hidden_states_before_norm=True, + kv_loc_plan=kv_loc_plan, ) self._apply_deferred_mamba_init_to_draft_pools(forward_batch) @@ -766,14 +778,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): draft_token_num=self.speculative_num_draft_tokens, device=batch.device, ) + seq_lens = batch.seq_lens + self.speculative_num_draft_tokens runner.stage_shared_reads( - seq_lens=batch.seq_lens + self.speculative_num_draft_tokens, + seq_lens=seq_lens, req_pool_indices=batch.req_pool_indices, out_cache_loc=locs, positions=positions, - # Both loc sources read req_to_token untranslated, so `locs` is the - # PRE-translate write loc the capture rail needs. + # Both loc sources read req_to_token untranslated: `locs` is virtual. out_cache_loc_virtual=locs, + kv_loc_plan=self.draft_runner_list[0].kv_index_translator.plan( + req_pool_indices=batch.req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=None, + write_virtual=locs, + ), ) return True @@ -783,6 +801,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): batch_result: GenerationBatchResult, *, staged: bool = False, + kv_loc_plan: Optional[KVLocPlan] = None, ): # Batch 2: Draft extend draft_extend_input = EagleDraftExtendInput( @@ -826,6 +845,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): return_hidden_states_before_norm=True, widened_out_cache_loc=boundary_kv_locs, widened_positions=boundary_kv_positions, + kv_loc_plan=kv_loc_plan, ) if self.plan_stream: @@ -1091,7 +1111,9 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker): batch, pp_proxy_tensors=pp_proxy_tensors, capture_hidden_mode=target_capture_mode, + return_kv_loc_plan=True, ) + kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None # Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens. # Extend processed L prompt tokens; next verify iter expects same L. @@ -1106,6 +1128,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker): batch, batch_output.logits_output.hidden_states, batch_output.next_token_ids, + kv_loc_plan=kv_loc_plan, ) return batch_output else: @@ -1134,7 +1157,10 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker): if on_publish is not None: on_publish(batch_output.new_seq_lens) self.draft_worker._draft_extend_for_decode( - batch, batch_output, staged=staged + batch, + batch_output, + staged=staged, + kv_loc_plan=verify_input.kv_loc_plan, ) return batch_output diff --git a/python/sglang/srt/speculative/ngram_info.py b/python/sglang/srt/speculative/ngram_info.py index ce5d7cbabe..e0309fb3d3 100644 --- a/python/sglang/srt/speculative/ngram_info.py +++ b/python/sglang/srt/speculative/ngram_info.py @@ -5,6 +5,7 @@ from typing import List, Optional import torch from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan from sglang.srt.speculative.spec_info import SpecInput, SpecInputType @@ -63,11 +64,12 @@ class NgramVerifyInput(SpecInput): paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, translator: KVIndexTranslator, - sliding_window: bool = False, + plan: KVLocPlan, + kind: IdSpaceKind = IdSpaceKind.FULL, ): - """CSR verify args, gathered straight into the packed stream. The lens - are widened here and handed to the translator, so nothing materializes - a ``[bs, max_pages]`` rectangle to repack from.""" + """CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and + is widened here; the translator packs the read ids straight into + ``kv_indices``.""" bs = req_pool_indices.numel() cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device) @@ -83,13 +85,13 @@ class NgramVerifyInput(SpecInput): total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device) - translator.fill_packed_read_stream( + translator.pack_read_stream( + plan, req_pool_indices=req_pool_indices, seq_lens=paged_kernel_lens, indptr=cum_kv_seq_len, - total_tokens=total_tokens, out=kv_indices, - sliding_window=sliding_window, + kind=kind, ) # Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests. diff --git a/python/sglang/srt/speculative/uno_worker_v2.py b/python/sglang/srt/speculative/uno_worker_v2.py index 472de85cec..d5aff14fe9 100644 --- a/python/sglang/srt/speculative/uno_worker_v2.py +++ b/python/sglang/srt/speculative/uno_worker_v2.py @@ -297,7 +297,7 @@ class UnoWorkerV2(BaseSpecWorker): draft_token_num=self.forward_width, ) - return ForwardBatch( + forward_batch = ForwardBatch( forward_mode=ForwardMode.TARGET_VERIFY, batch_size=len(prefix_lens), input_ids=input_ids, @@ -312,6 +312,8 @@ class UnoWorkerV2(BaseSpecWorker): capture_hidden_mode=CaptureHiddenMode.NULL, return_hidden_states_before_norm=False, ) + self.model_runner.kv_index_translator.bind_own_plan(forward_batch) + return forward_batch def _run_target_block( self, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py index 3b0f392c5b..9368665686 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dense_attention.py @@ -761,6 +761,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): batch.extend_prefix_lens = torch.tensor( diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py index 7ddef6953e..c23a40c898 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py @@ -26,6 +26,7 @@ from sglang.srt.environ import envs from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.model_executor.cuda_graph_config import ( Backend, @@ -419,6 +420,13 @@ class MockDSV4ModelRunner: identity = torch.arange(swa_size, dtype=torch.int64, device=device) self.token_to_kv_pool.register_mapping(identity) self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size) + self.kv_index_translator = KVIndexTranslator( + req_to_token=self.req_to_token_pool.req_to_token, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + token_to_kv_pool=self.token_to_kv_pool, + page_size=case.page_size, + device=device, + ) self.attention_chunk_size = None self.hisparse_coordinator = None self.init_new_workspace = False @@ -661,6 +669,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) # extend_* fields are only populated for extend-shaped modes. DECODE leaves # them at their defaults; the flash_mla path reads metadata directly from # DSV4AttnMetadata so the extend fields are unused for the compress_ratio=0 diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py index af2f7d7005..1fd86b1e1b 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/gdn_attention.py @@ -322,6 +322,7 @@ class MockGDNModelRunner(ModelRunner): page_size=case.page_size, get_kvcache=lambda: self.token_to_kv_pool, ) + self.is_draft_worker = False self.init_kv_index_translator() self.attention_chunk_size = None self.hisparse_coordinator = None @@ -329,7 +330,6 @@ class MockGDNModelRunner(ModelRunner): self.is_hybrid_swa = False self.sliding_window_size = None self.use_mla_backend = False - self.is_draft_worker = False self._kernel_warmed_up = True @property @@ -562,6 +562,8 @@ def _make_forward_batch( else None ), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py index ddd0fb9f98..efc2bacace 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/kda_attention.py @@ -318,6 +318,7 @@ class MockKDAModelRunner(ModelRunner): enable_alt_stream=False, ) self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size) + self.is_draft_worker = False self.init_kv_index_translator() self.attention_chunk_size = None self.hisparse_coordinator = None @@ -325,7 +326,6 @@ class MockKDAModelRunner(ModelRunner): self.is_hybrid_swa = False self.sliding_window_size = None self.use_mla_backend = False - self.is_draft_worker = False self._kernel_warmed_up = True @property @@ -552,6 +552,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py index eafef40af8..d3d8206a47 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py @@ -327,6 +327,7 @@ class MockLightningModelRunner(ModelRunner): enable_alt_stream=False, ) self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size) + self.is_draft_worker = False self.init_kv_index_translator() self.attention_chunk_size = None self.hisparse_coordinator = None @@ -334,7 +335,6 @@ class MockLightningModelRunner(ModelRunner): self.is_hybrid_swa = False self.sliding_window_size = None self.use_mla_backend = False - self.is_draft_worker = False self._kernel_warmed_up = True @property @@ -523,6 +523,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py index cf4589efc2..2e44ea402d 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mamba2_attention.py @@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner): enable_alt_stream=False, ) self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size) + self.is_draft_worker = False self.init_kv_index_translator() self.attention_chunk_size = None self.hisparse_coordinator = None @@ -460,7 +461,6 @@ class MockMamba2ModelRunner(ModelRunner): self.is_hybrid_swa = False self.sliding_window_size = None self.use_mla_backend = False - self.is_draft_worker = False self._kernel_warmed_up = True @property @@ -653,6 +653,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py index d535c3d0ac..132c7f1bbc 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py @@ -307,6 +307,7 @@ class MockMLAModelRunner(ModelRunner): enable_memory_saver=False, ) self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size) + self.is_draft_worker = False self.init_kv_index_translator() self.attention_chunk_size = None self.hisparse_coordinator = None @@ -314,7 +315,6 @@ class MockMLAModelRunner(ModelRunner): self.is_hybrid_swa = False self.sliding_window_size = None self.use_mla_backend = True - self.is_draft_worker = False self._kernel_warmed_up = True # Runner-mode helpers mutate speculative graph sizes after construction. self.graph_shared_output = GraphSharedOutput( @@ -693,6 +693,8 @@ def _make_forward_batch( seq_lens_sum=sum(seq_lens), positions=torch.tensor(positions, dtype=torch.int64, device=device), ) + # Production batches take their KV ids from a plan (`init_new`). + runner.kv_index_translator.bind_own_plan(batch) if case.forward_mode.is_extend(include_draft_extend_v2=True): extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device) diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/cuda_graph_decode_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/cuda_graph_decode_runner.py index b702744d1c..20d560ed84 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/cuda_graph_decode_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/cuda_graph_decode_runner.py @@ -297,6 +297,8 @@ def _init_cuda_graph_replay_metadata(backend, capture_batch_size: int, batch): encoder_lens=batch.encoder_lens, extend_seq_lens=getattr(batch, "extend_seq_lens", None), out_cache_loc=getattr(batch, "out_cache_loc", None), + kv_loc_plan=batch.kv_loc_plan, + kv_loc_cols=batch.kv_loc_cols, spec_info=batch.spec_info, ) backend.init_forward_metadata_out_graph(fb_view) diff --git a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py index 802a677dfe..23fcb5ef1f 100644 --- a/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py +++ b/python/sglang/test/kits/attention_unittest/runner_modes/speculative_draft_runner.py @@ -510,6 +510,7 @@ def run_eagle_draft_cuda_graph_runner_case( ) adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings) eager_batch = adapter.make_forward_batch(case, draft_inputs, settings) + eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch) expected = _run_eagle_draft_eager( eager_worker, eager_batch, @@ -526,6 +527,7 @@ def run_eagle_draft_cuda_graph_runner_case( ) adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings) graph_batch = adapter.make_forward_batch(case, draft_inputs, settings) + graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch) graph_runner = _capture_eagle_draft_graph_runner( graph_worker, graph_backend, @@ -565,6 +567,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case( ) adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings) eager_batch = adapter.make_forward_batch(case, draft_inputs, settings) + eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch) expected = _run_frozen_kv_mtp_eager(eager_worker, eager_batch) graph_fixture, graph_worker, _ = _build_frozen_kv_mtp_fixture( @@ -576,6 +579,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case( ) adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings) graph_batch = adapter.make_forward_batch(case, draft_inputs, settings) + graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch) graph_runner = _capture_frozen_kv_mtp_graph_runner(graph_worker) adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings) diff --git a/test/registered/attention/test_gemma4_unified_swa_virtual_ids.py b/test/registered/attention/test_gemma4_unified_swa_virtual_ids.py index 4b43774406..90ab7252d1 100644 --- a/test/registered/attention/test_gemma4_unified_swa_virtual_ids.py +++ b/test/registered/attention/test_gemma4_unified_swa_virtual_ids.py @@ -19,8 +19,8 @@ FULL side's ids while it used to be sized by its own page count. A model with few full-attention layers and many sliding ones gives the id owner more pages, and the top of that id space then lands off the end of the swa table. On GPU the write is unchecked (`alloc_bind_inplace`'s `tl.store`), so the symptom -surfaces later and elsewhere: a device-side index assert on the read in -`_swa_write_loc_unified`, and a dead scheduler. +surfaces later and elsewhere: a device-side index assert on a later read of +the swa table, and a dead scheduler. The reachability of it is a KV-budget property, not just a model property: ids come off the front of the owner's free list and freed ids return to the back, so diff --git a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py index 8f52551df4..f8296a0efc 100644 --- a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py +++ b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py @@ -55,6 +55,7 @@ def _apply(backend, *, bs: int, seq_lens: torch.Tensor, forward_mode): patch.object(backend, "_fill_dcp_block_kv_indices") as fill, ): backend._apply_cuda_graph_metadata( + plan=None, # DCP builds its own block table bs=bs, req_pool_indices=torch.arange(bs, dtype=torch.int32, device="cuda"), seq_lens=seq_lens, @@ -108,7 +109,6 @@ class _DCPMetadataTests: heads, v_head_dim, n = 4, 512, 3 * NUM_DRAFT_TOKENS backend = object.__new__(self.backend_cls) backend.data_type = backend.q_data_type = torch.bfloat16 - backend._decode_kernel_loc = None def real_path(*args, **kwargs): raise _RealVerifyPath diff --git a/test/registered/e2e/models/test_kimi_linear_unified_memory.py b/test/registered/e2e/models/test_kimi_linear_unified_memory.py index 72c907316c..bb0405df0d 100644 --- a/test/registered/e2e/models/test_kimi_linear_unified_memory.py +++ b/test/registered/e2e/models/test_kimi_linear_unified_memory.py @@ -24,7 +24,7 @@ class TestKimiLinearUnifiedMemoryFlashMLA( GSM8KMixin, PrefixCacheBranchingMixin, DefaultServerBase ): """flashmla at its ps=64 snap: the block-table route - (KVIndexTranslator.fill_read_table into flashmla's padded tables) plus the + (the plan's page table copied into flashmla's padded tables) plus the ps=64 sub-pool sizing (64-token sink floor, per-layer-view tail pad). Hopper-only, like the rest of this nightly suite.""" diff --git a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py index a53395082d..554ef6bd4d 100644 --- a/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py +++ b/test/registered/kernels/ops/attention/test_inkling_checkpoint_indices.py @@ -12,6 +12,7 @@ from sglang.srt.layers.attention.linear.inkling_sconv_backend import ( InklingShortConvHybridAttnBackend, ) from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool from sglang.srt.mem_cache.unified_memory_pool import ( MambaSubPoolSpec, @@ -461,6 +462,9 @@ class TestInklingCheckpointIndices(CustomTestCase): composite.buffers = buffers composite.captured_req_width = 4 composite.draft_extend_attn_backend_list = [wrapper] + composite._out_cache_loc_virtual = None + composite._kv_loc_plan = None + composite._kv_loc_cols = None for step, tracking in enumerate((True, False, True)): with self.subTest(tracking=tracking, step=step): ids = slots[-4:].roll(step).clone() @@ -814,6 +818,15 @@ class TestInklingCheckpointIndices(CustomTestCase): spec_algorithm=SimpleNamespace(is_standalone=lambda: False), device_timer=None, canary_manager=None, + # The KV side is not under test: a static pool's translator binds + # the captured batch to its own write ids. + kv_index_translator=KVIndexTranslator( + req_to_token=torch.zeros((1, 8), dtype=torch.int32, device="cuda"), + token_to_kv_pool_allocator=None, + token_to_kv_pool=object(), + page_size=1, + device="cuda", + ), ) return runner, backend, allocator, pool, slots, buffers @@ -926,6 +939,9 @@ class TestInklingCheckpointIndices(CustomTestCase): input_ids=buffers.input_ids[: live * 4], positions=buffers.positions[: live * 4], out_cache_loc=buffers.out_cache_loc[: live * 4], + out_cache_loc_virtual=None, + kv_loc_plan=None, + kv_loc_cols=None, req_pool_indices=torch.arange(1, live + 1, device="cuda"), seq_lens=torch.full((live,), 6, dtype=torch.int64, device="cuda"), seq_lens_sum=6 * live, @@ -973,7 +989,13 @@ class TestInklingCheckpointIndices(CustomTestCase): composite._stage_metadata = Mock() composite.prepare(fresh) runner.replay( - 4, composite.seq_lens_sum, composite._replay_spec_info, None + 4, + composite.seq_lens_sum, + composite._replay_spec_info, + None, + out_cache_loc_virtual=composite._out_cache_loc_virtual, + kv_loc_plan=composite._kv_loc_plan, + kv_loc_cols=composite._kv_loc_cols, ) else: runner.execute(fresh, torch.arange(live, device="cuda") * 4 + 1) diff --git a/test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py b/test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py new file mode 100644 index 0000000000..97175e9c24 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py @@ -0,0 +1,126 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`generate_draft_decode_kv_indices` over the unified pool's read tables. + +Over the plan's physical page table (token ids rebuilt as +`entry * ps + pos % ps`), and over virtual rows translated in the kernel's own +gather, the kernel must emit exactly the translated virtual ids and the same +kv_indptr. The static `req_to_token` path is covered by +test_spec_kv_indices_grid.py. + + python -m pytest test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py -v +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_SENTINEL = -7 + + +def _run_kernel( + *, table, entry_page_size, seq_lens, positions, num_steps, topk, page_size, v2p=None +): + from sglang.kernels.ops.speculative.cache_locs import ( + generate_draft_decode_kv_indices, + ) + from sglang.srt.utils import next_power_of_2 + + num_seqs = seq_lens.numel() + bs = num_seqs * topk + width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64) + kv_indices = torch.full( + (num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda" + ) + kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda") + generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)]( + torch.arange(num_seqs, dtype=torch.int64, device="cuda"), + table, + seq_lens, + kv_indices, + kv_indptr, + positions, + table.stride(0), + width, + kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(num_steps), + next_power_of_2(bs), + page_size, + ENTRY_PAGE_SIZE=entry_page_size, + v2p=v2p, + TRANSLATE=v2p is not None, + ) + return kv_indices, kv_indptr + + +class TestDraftDecodeKVIndicesReadTable(CustomTestCase): + def _check(self, *, page_size, num_steps=3, topk=1): + torch.manual_seed(7) + num_seqs, max_context = 3, 64 + num_pages = max_context // page_size + # Each row holds whole virtual pages in a shuffled order, and the + # virtual->physical table is a shuffle too, so nothing is identity. + v2p = torch.randperm(num_pages, device="cuda") + rows = [] + for _ in range(num_seqs): + vpages = torch.randperm(num_pages, device="cuda") + offsets = torch.arange(page_size, device="cuda") + rows.append((vpages[:, None] * page_size + offsets).reshape(-1)) + req_to_token = torch.stack(rows) + # The plan's table: one physical page per virtual page of the row. + table = v2p[req_to_token[:, ::page_size] // page_size].to(torch.int32) + seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda") + positions = seq_lens.repeat_interleave(topk) + common = dict( + seq_lens=seq_lens, + positions=positions, + num_steps=num_steps, + topk=topk, + page_size=page_size, + ) + raw, raw_indptr = _run_kernel(table=req_to_token, entry_page_size=1, **common) + out, out_indptr = _run_kernel(table=table, entry_page_size=page_size, **common) + # No shared table: the virtual rows, translated in the kernel's gather. + gathered, gathered_indptr = _run_kernel( + table=req_to_token, entry_page_size=1, v2p=v2p, **common + ) + + torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0) + written = raw != _SENTINEL + self.assertTrue(bool((out[~written] == _SENTINEL).all())) + virt = raw[written] + expected = v2p[virt // page_size] * page_size + virt % page_size + torch.testing.assert_close(out[written], expected, rtol=0, atol=0) + torch.testing.assert_close(gathered_indptr, out_indptr, rtol=0, atol=0) + torch.testing.assert_close(gathered, out, rtol=0, atol=0) + + def test_page_size_one(self): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + self._check(page_size=1) + + def test_page_size_two(self): + if not torch.cuda.is_available(): + self.skipTest("CUDA required") + self._check(page_size=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_draft_decode_window.py b/test/registered/kernels/ops/speculative/test_draft_decode_window.py index f214911d71..0617d295f7 100644 --- a/test/registered/kernels/ops/speculative/test_draft_decode_window.py +++ b/test/registered/kernels/ops/speculative/test_draft_decode_window.py @@ -117,7 +117,6 @@ def _run(io, window_size=0, sink_size=0): kv_indices, kv_indptr, io["positions"], - None, # v2p: no unified-pool translation io["pool_len"], kv_indices.shape[1], kv_indptr.shape[1], diff --git a/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py b/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py deleted file mode 100644 index ad66497b3f..0000000000 --- a/test/registered/kernels/ops/speculative/test_generate_draft_decode_kv_indices_translate.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2023-2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""`generate_draft_decode_kv_indices` under the unified pool's translation. - -BUG REGRESSION. The multi-step draft-decode kernel copied req_to_token values -verbatim into the per-step kv_indices. Under `--enable-unified-memory` those -values are VIRTUAL ids while the fused draft pool is indexed by physical -ids, so with `speculative_num_steps > 1` every draft decode step read the -wrong KV rows -- silently: bad drafts only depress acceptance, verify stays -correct. Pinned here: TRANSLATE=True must emit exactly `translate_kv_loc`'s -physical ids, and TRANSLATE=False must stay byte-identical to the historical -raw copy. - - python -m pytest test/registered/kernels/test_generate_draft_decode_kv_indices_translate.py -v -""" - -import unittest - -import torch - -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") - -_SENTINEL = -7 - - -def _run_kernel( - *, - req_to_token, - seq_lens, - positions, - num_steps, - topk, - page_size, - v2p, - translate, -): - from sglang.kernels.ops.speculative.cache_locs import ( - generate_draft_decode_kv_indices, - ) - from sglang.srt.utils import next_power_of_2 - - num_seqs = seq_lens.numel() - bs = num_seqs * topk - width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64) - kv_indices = torch.full( - (num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda" - ) - kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda") - generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)]( - torch.arange(num_seqs, dtype=torch.int64, device="cuda"), - req_to_token, - seq_lens, - kv_indices, - kv_indptr, - positions, - v2p if translate else None, - req_to_token.shape[1], - width, - kv_indptr.shape[1], - next_power_of_2(num_seqs), - next_power_of_2(num_steps), - next_power_of_2(bs), - page_size, - TRANSLATE=translate, - ) - return kv_indices, kv_indptr - - -class TestGenerateDraftDecodeKVIndicesTranslate(CustomTestCase): - def _check(self, *, page_size, num_steps=3, topk=1, unused_slots=False): - torch.manual_seed(7) - num_seqs, max_context = 3, 64 - num_pages = max_context // page_size + 1 - # Virtual ids: a shuffled page layout so translation is not identity. - perm = torch.randperm(num_pages, device="cuda") - v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda") - v2p[perm] = torch.arange(num_pages, device="cuda") - req_to_token = ( - torch.arange(max_context, dtype=torch.int64, device="cuda") - .flip(0) - .repeat(num_seqs, 1) - ) - if unused_slots: - # Negative entries inside the read window must land on the sink. - req_to_token[:, ::3] = -1 - seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")[:num_seqs] - positions = seq_lens.repeat_interleave(topk) - - common = dict( - req_to_token=req_to_token, - seq_lens=seq_lens, - positions=positions, - num_steps=num_steps, - topk=topk, - page_size=page_size, - v2p=v2p, - ) - raw, raw_indptr = _run_kernel(translate=False, **common) - out, out_indptr = _run_kernel(translate=True, **common) - - torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0) - written = raw != _SENTINEL - # Unwritten lanes stay untouched in both compilations. - self.assertTrue(bool((out[~written] == _SENTINEL).all())) - # Written lanes: translate_kv_loc's formula over the raw ids. - virt = raw[written] - page = torch.where(virt < 0, 0, virt // page_size) - expected = torch.where( - virt < 0, - 0, - torch.clamp_min(v2p[page] * page_size + virt % page_size, 0), - ) - torch.testing.assert_close(out[written], expected, rtol=0, atol=0) - - def test_page_size_one(self): - if not torch.cuda.is_available(): - self.skipTest("CUDA required") - self._check(page_size=1) - - def test_page_size_two(self): - if not torch.cuda.is_available(): - self.skipTest("CUDA required") - self._check(page_size=2) - - def test_negative_ids_map_to_the_sink(self): - if not torch.cuda.is_available(): - self.skipTest("CUDA required") - for page_size in (1, 2): - self._check(page_size=page_size, unused_slots=True) - - def test_translate_false_is_a_raw_copy(self): - """The static-pool compilation must stay byte-identical to the - historical kernel: the emitted indices ARE the req_to_token values.""" - if not torch.cuda.is_available(): - self.skipTest("CUDA required") - torch.manual_seed(11) - num_seqs, max_context, page_size = 2, 32, 1 - req_to_token = torch.randint( - 0, 1 << 20, (num_seqs, max_context), dtype=torch.int64, device="cuda" - ) - seq_lens = torch.tensor([4, 7], dtype=torch.int64, device="cuda") - raw, _ = _run_kernel( - req_to_token=req_to_token, - seq_lens=seq_lens, - positions=seq_lens, - num_steps=2, - topk=1, - page_size=page_size, - v2p=None, - translate=False, - ) - written = raw != _SENTINEL - emitted = raw[written] - self.assertTrue(bool(torch.isin(emitted, req_to_token.flatten()).all())) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py index dece431904..d93ad83dee 100644 --- a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py +++ b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py @@ -60,7 +60,6 @@ def _run_draft(kern_inputs, topk, steps, page_size, nb, kw): kv_i, kv_p, positions, - None, # v2p: no unified-pool translation POOL_LEN, kv_i.shape[1], kv_p.shape[1], diff --git a/test/registered/unit/dllm/test_gemma4_cuda_graph.py b/test/registered/unit/dllm/test_gemma4_cuda_graph.py index 4198b5ae93..41506a6a00 100644 --- a/test/registered/unit/dllm/test_gemma4_cuda_graph.py +++ b/test/registered/unit/dllm/test_gemma4_cuda_graph.py @@ -9,6 +9,7 @@ import torch from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise from sglang.srt.layers.attention.triton_backend import TritonAttnBackend from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.runner.decode_cuda_graph_runner import ( DecodeCudaGraphRunner, @@ -26,16 +27,16 @@ class _ReadStream: def __init__(self): self.tokens = torch.arange(64).view(4, 16) - def fill_packed_read_stream( + def pack_read_stream( self, + plan, *, req_pool_indices, seq_lens, indptr, - total_tokens, out, kv_start_idx=None, - sliding_window=False, + kind=IdSpaceKind.FULL, token_mapping=None, ): for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)): @@ -91,6 +92,7 @@ class TestGemma4GraphMetadata(unittest.TestCase): requests = torch.tensor([1, 2, 0]) # Prefixes 3, 11, and an empty padded request; each has a four-token canvas. backend._apply_cuda_graph_metadata( + plan=None, bs=3, req_pool_indices=requests, seq_lens=torch.tensor([7, 15, 4]), @@ -126,6 +128,7 @@ class TestGemma4GraphMetadata(unittest.TestCase): metadata.window_kv_indices.data_ptr(), ) backend._apply_cuda_graph_metadata( + plan=None, bs=3, req_pool_indices=torch.tensor([3, 1, 0]), seq_lens=torch.tensor([4, 9, 4]), diff --git a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py index d9f8fb4e58..0e63e94f6b 100644 --- a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py +++ b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py @@ -89,10 +89,8 @@ class TestFlashAttentionGraphMetadata(CustomTestCase): class TestSpecReadSeqLenDelta(CustomTestCase): - """The mode -> seq_len_delta map IS the widening the whole-sequence spec - reads apply to cache_seqlens; a drift silently truncates the verify or - draft tail of the translated read table (the metadata builders widen - cache_seqlens while the source fill stays prefix-only).""" + """How far past seq_lens each spec mode's translated read table reaches; + too short silently leaves the verify or draft tail stale.""" def _backend(self, *, topk=1, num_steps=3, step_id=1, num_draft=4): b = FlashAttentionBackend.__new__(FlashAttentionBackend) @@ -127,11 +125,8 @@ class TestSpecReadSeqLenDelta(CustomTestCase): class TestDraftExtendInGraph(CustomTestCase): - """The draft-extend cuda-graph runners record - `init_forward_metadata_in_graph` unconditionally. Under translation fa3 - rebuilds the draft-extend tables out of graph on every replay, so the - in-graph gather -- raw req_to_token, i.e. virtual ids -- must not be - recorded, or each replay overwrites the translated tables.""" + """A translating fa3 must not record the in-graph draft-extend gather: it + reads raw (virtual) req_to_token ids over the tables rebuilt out of graph.""" def _run(self, *, translating): b = FlashAttentionBackend.__new__(FlashAttentionBackend) @@ -140,10 +135,8 @@ class TestDraftExtendInGraph(CustomTestCase): fb = SimpleNamespace(forward_mode=ForwardMode.DRAFT_EXTEND_V2, batch_size=2) b.init_forward_metadata_in_graph(fb) - def test_translating_backend_records_nothing(self): + def test_only_a_static_backend_builds_in_graph(self): self._run(translating=True) - - def test_static_backend_still_builds_in_graph(self): with self.assertRaises(KeyError): self._run(translating=False) diff --git a/test/registered/unit/layers/attention/test_kv_translate_ownership.py b/test/registered/unit/layers/attention/test_kv_translate_ownership.py index 97882643e4..b82b394482 100644 --- a/test/registered/unit/layers/attention/test_kv_translate_ownership.py +++ b/test/registered/unit/layers/attention/test_kv_translate_ownership.py @@ -1,9 +1,10 @@ """Nothing under layers/attention may translate KV ids for itself. -Ownership is exactly two places: `KVIndexTranslator` for READS and the -ForwardBatch rebind (`rebind_write_loc`) for WRITES. Virtual and physical ids -share a value range, so a backend that forgets a translate -- or does one -twice -- reads the wrong rows and nothing crashes. +Ownership is exactly one place: the iteration's `KVLocPlan`, built through +`KVIndexTranslator`, translates the WRITE window and the READ table once, and +backends read the result. Virtual and physical ids share a value range, so a +backend that forgets a translate -- or does one twice -- reads the wrong rows +and nothing crashes. Deliberately out of scope: the allocator-internal implementations (`allocator/unified_*`, `unified_memory_pool`), which ARE the mechanism the diff --git a/test/registered/unit/layers/attention/test_triton_verify_metadata.py b/test/registered/unit/layers/attention/test_triton_verify_metadata.py index fc3a46b864..4b412706f2 100644 --- a/test/registered/unit/layers/attention/test_triton_verify_metadata.py +++ b/test/registered/unit/layers/attention/test_triton_verify_metadata.py @@ -9,6 +9,7 @@ from sglang.srt.layers.attention.triton_backend import ( TritonAttnBackend, ) from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.speculative.spec_info import SpecInput, SpecInputType from sglang.test.ci.ci_register import register_cpu_ci @@ -26,16 +27,19 @@ class _BonusTokenVerifyInput(SpecInput): class _KVIndexTranslator: is_translating = False - def fill_packed_read_stream( + def pack_read_stream( self, + plan, *, req_pool_indices, seq_lens, indptr, - total_tokens, out, + kv_start_idx=None, + kind=IdSpaceKind.FULL, ): out.zero_() + return False class _RecordingTritonBackend(TritonAttnBackend): @@ -92,6 +96,8 @@ def _make_forward_batch(batch_size, spec_info): spec_info=spec_info, forward_mode=ForwardMode.TARGET_VERIFY, out_cache_loc=torch.zeros(batch_size * 7, dtype=torch.int64), + # A static pool's reads never touch the plan. + kv_loc_plan=None, ) diff --git a/test/registered/unit/layers/attention/test_verify_widening_is_wired.py b/test/registered/unit/layers/attention/test_verify_widening_is_wired.py index 7c66f92a85..e70a5a0258 100644 --- a/test/registered/unit/layers/attention/test_verify_widening_is_wired.py +++ b/test/registered/unit/layers/attention/test_verify_widening_is_wired.py @@ -15,14 +15,10 @@ A whole-sequence verify reads `[committed prefix | drafts]` back out of the pool, so under the unified pool the translated page table a verify kernel -reads has to cover `seq_lens + num_draft_tokens` columns per row. A table -filled only to `seq_lens` leaves the draft tail stale: invisible at -page_size 256, where the un-widened `ceil(seq/ps)` pages still cover a few -draft tokens, and at page_size 1 observed as accept length collapsing -(6.20 -> 1.37) or wrong tokens (gsm8k 0.905 -> 0.730). +reads has to cover `seq_lens + num_draft_tokens` columns per row; a table +filled only to `seq_lens` silently leaves the draft tail stale. -These tests drive fa3's two verify builds -- the eager one and the captured -one a cuda-graph replay runs -- through a translating KV-index translator +These tests drive fa3's eager and captured verify builds from a verify plan over a non-identity virtual->physical page map, and check the page table the kernel reads: every row translated, and filled through the draft tail. @@ -39,6 +35,7 @@ from sglang.srt.layers.attention.flashattention_backend import ( FlashAttentionMetadata, ) from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -67,9 +64,16 @@ def _translator(req_to_token, page_size, n_pages): t.defer_read_translate = False t._capture_page_size = page_size t._full_v2p_table = v2p.to(_DEV) - t._swa_v2p_table = None + t._spaces = { + IdSpaceKind.FULL: IdSpace( + key=(IdSpaceKind.FULL, "test"), + write=lambda ids: ( + t._full_v2p_table[ids // page_size] * page_size + ids % page_size + ), + read_v2p=t._full_v2p_table, + ) + } t._rows = torch.arange(req_to_token.shape[0], dtype=torch.int64, device=_DEV) - t._index_table_memo = None return t @@ -99,6 +103,25 @@ def _expected(req_to_token, live, v2p, page_size): return rows +def _verify_plan(translator, req_to_token, live, host_lens=None): + """The plan a verify forward builds for itself: it writes the draft window + past `live`, so it reads that far.""" + bs = live.numel() + rows = torch.arange(bs, device=_DEV) + cols = live.to(_DEV)[:, None] + torch.arange(_NUM_DRAFT, device=_DEV)[None, :] + return translator.own_plan( + SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + spec_info=SimpleNamespace(draft_token_num=_NUM_DRAFT), + batch_size=bs, + req_pool_indices=rows, + seq_lens=live.to(_DEV), + seq_lens_cpu=live if host_lens is None else host_lens, + out_cache_loc=req_to_token[rows[:, None], cols].long().reshape(-1), + ) + ) + + def _backend(translator, req_to_token, page_size): b = FlashAttentionBackend.__new__(FlashAttentionBackend) b.kv_index_translator = translator @@ -153,6 +176,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase): metadata.swa_page_table = None b.target_verify_metadata = {bs: metadata} b._apply_cuda_graph_metadata( + _verify_plan(translator, req_to_token, live), bs, torch.arange(bs, device=_DEV), live.to(_DEV), @@ -193,6 +217,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase): spec_info=spec_info, encoder_lens=None, out_cache_loc=None, + kv_loc_plan=_verify_plan(translator, req_to_token, live, host_lens), ) b.init_forward_metadata(fb) self._check( diff --git a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py index 6dc5098465..9f3e5a4311 100644 --- a/test/registered/unit/layers/test_mamba_prefill_track_metadata.py +++ b/test/registered/unit/layers/test_mamba_prefill_track_metadata.py @@ -225,7 +225,12 @@ class TestMambaPrefillTrackMetadata(unittest.TestCase): model_config=SimpleNamespace( requires_mm_token_modalities=False, model_is_mrope=False ), - kv_index_translator=SimpleNamespace(rebind_write_loc=lambda forward: None), + # A pass-through runner: its plan hands the batch its ids untouched. + kv_index_translator=SimpleNamespace( + own_plan=lambda batch: SimpleNamespace( + bind=lambda batch, reader, cols=None: None + ) + ), prefill_attention_backend_str="torch_native", ngram_embedding_manager=SimpleNamespace(enabled=False), lora_manager=None, diff --git a/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py index 3a192d9acb..3b67db423a 100644 --- a/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py +++ b/test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py @@ -1,17 +1,12 @@ """Dense fused-draft geometry: the draft model's KV as two more parts of every -host slot's entry. - -Derived-property pins for the fused entry layout +host slot's entry, [ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ] -the draft parts start at the 16-B-aligned end of the host parts and the entry -rounds up to the 32-B entry alignment (never to an lcm of row widths), so the -draft's row width is free to differ from the host's; host and draft views -share the slot stride and are indexed by the same physical token id; and +the draft parts starting at the aligned end of the host parts. Host and draft +views share the slot stride and are indexed by the same physical token id; writes through either family's views land inside their own part of their own -slot (compaction moves whole page envelopes, so confinement IS the correctness -of the fused move). +slot, and a host page move carries the draft bytes with it. python -m pytest test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py -v """ @@ -27,7 +22,6 @@ from sglang.srt.mem_cache.layout.fused_draft import ( ) from sglang.srt.mem_cache.layout.token_major import ( ENTRY_ALIGN_BYTES, - ROW_ALIGN_BYTES, align_entry_bytes, align_part_offset, ) @@ -43,11 +37,10 @@ from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu") +register_cpu_ci(est_time=7, suite="base-a-test-cpu") _DEV = "cpu" _DTYPE = torch.bfloat16 -_ITEM = _DTYPE.itemsize def _host_spec(draft_region=None, layer_num=2, head_num=2, head_dim=4): @@ -75,12 +68,6 @@ def _placement(region): class TestFusedSpecMath(unittest.TestCase): - def test_unfused_spec_is_byte_identical_to_before(self): - s = _host_spec() - self.assertEqual(s.entry_bytes(), 2 * (16 + 16)) - self.assertEqual(s.host_entry_bytes(), s.entry_bytes()) - self.assertEqual([p.name for p in s.layout().parts], ["k", "v"]) - def test_fused_entry_appends_the_draft_parts(self): f = _host_spec(_draft_region()) r = f.draft_region @@ -101,46 +88,6 @@ class TestFusedSpecMath(unittest.TestCase): self.assertEqual(layout.part("draft_v").offset_bytes, 64 + 48) self.assertEqual(layout.part("draft_k").layer_stride_bytes, 64) - def test_entry_alignment_pads_only_to_the_entry_quantum(self): - # host 32 B + draft 48 B = 80 B of rows -> one 96 B entry (16 B pad), - # not an lcm of the two row widths. - f = _host_spec( - DenseDraftRegion( - lane_num=1, head_num=1, head_dim=8, v_head_dim=16, store_dtype=_DTYPE - ), - layer_num=1, - head_num=1, - head_dim=8, - ) - self.assertEqual(f.host_entry_bytes(), 32) - self.assertEqual(f.draft_region.entry_bytes(), 48) - self.assertEqual(f.entry_bytes(), 96) - self.assertEqual(f.draft_offset_in_entry() % ROW_ALIGN_BYTES, 0) - f.layout() # parts fit and do not overlap - - def test_draft_rows_must_be_row_aligned(self): - with self.assertRaises(AssertionError): - _host_spec( - DenseDraftRegion(lane_num=1, head_num=1, head_dim=3, store_dtype=_DTYPE) - ).layout() - - def test_only_page_envelope_kinds_accept_a_draft_region(self): - """`draft_region` is a universal spec field (None = unfused), but a - kind without the fused entry layout must refuse one at construction; - a silently-carried region would never reach the layout. MHA and MLA - entries carry the draft parts; mamba state pages carry none yet.""" - with self.assertRaises(AssertionError): - MambaSubPoolSpec( - name="mamba", - layer_num=1, - conv_state_shapes=((2, 2),), - conv_dtype=torch.float32, - temporal_state_shape=(2,), - temporal_dtype=torch.float32, - grow_direction="up", - draft_region=_draft_region(), - ) - class TestFusedRegionConfinement(unittest.TestCase): """Writes through host/draft views must land in their own part of their @@ -199,20 +146,6 @@ class TestFusedRegionConfinement(unittest.TestCase): self.assertGreaterEqual(lo, page * self.PS * entry) self.assertLessEqual(hi, (page + 1) * self.PS * entry) - def test_host_and_draft_views_share_the_slot_stride(self): - spec, _, (hk, hv), (dk, dv) = self._build() - stride = spec.entry_bytes() - for view in (*hk, *hv, *dk, *dv): - self.assertEqual(view.stride(0) * view.element_size(), stride) - self.assertEqual(view.shape[0], hk[0].shape[0]) - self.assertEqual(tuple(dk[0].shape[1:]), (1, 24)) - self.assertEqual(tuple(dv[0].shape[1:]), (1, 8)) - - def test_pool_helper_refuses_an_unfused_sub_pool(self): - _, pool, _, _ = self._build() - with self.assertRaises(AssertionError): - pool.build_dense_draft_views("swa") # no fused region there - class TestUnifiedDraftKVPool(unittest.TestCase): PS = 2 @@ -268,9 +201,9 @@ class TestUnifiedDraftKVPool(unittest.TestCase): ) def test_host_page_move_carries_the_draft_bytes(self): - # THE fused-layout property: compaction relocates whole page envelopes - # on the HOST pool; a draft marker written in page A must arrive at - # page B after host.move_kv_cache(B, A), with zero draft-side moves. + # Compaction relocates whole page envelopes on the HOST pool: a draft + # marker written in page A arrives at page B after + # host.move_kv_cache(B, A), with no draft-side move. pool = self._pool() dp, _ = self._draft_pool(pool) host = UnifiedMHATokenToKVPool( @@ -336,17 +269,6 @@ class TestUnifiedDraftKVPool(unittest.TestCase): self.assertEqual(dp.get_key_buffer(0).dtype, fp8) torch.testing.assert_close(dp.get_key_buffer(0)[loc].float(), k) - def test_draft_side_moves_and_transfers_fail_loudly(self): - pool = self._pool() - dp, _ = self._draft_pool(pool) - one = torch.zeros(self.PS, dtype=torch.int64) - with self.assertRaises(NotImplementedError): - dp.move_kv_cache(one, one) - with self.assertRaises(NotImplementedError): - dp.get_contiguous_buf_infos() - with self.assertRaises(NotImplementedError): - dp.get_cpu_copy(one) - def _mla_host_spec(draft_region=None): # 32 B latent rows, two layers: a 64 B host entry before the draft parts. @@ -376,8 +298,7 @@ def _mamba_spec(): class TestFusedMLAHost(unittest.TestCase): """MLA entries carry the fused draft parts exactly like MHA entries: the same aligned entry, one slot stride for both families, and byte-disjoint - host/draft parts within every slot. The unfused spec stays byte-identical; - that identity guards every existing MLA deploy.""" + host/draft parts within every slot.""" def setUp(self): # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived @@ -389,13 +310,6 @@ class TestFusedMLAHost(unittest.TestCase): PS = 2 PAGES = 8 - def test_unfused_spec_is_byte_identical_to_before(self): - s = _mla_host_spec() - self.assertEqual(s.row_bytes(), 32) - self.assertEqual(s.entry_bytes(), 64) - self.assertEqual(s.host_entry_bytes(), s.entry_bytes()) - self.assertEqual([p.name for p in s.layout().parts], ["kv"]) - def test_fused_entry_appends_the_draft_parts(self): f = _mla_host_spec(_draft_region()) self.assertEqual(f.host_entry_bytes(), 64) @@ -448,10 +362,9 @@ class TestFusedMLAHost(unittest.TestCase): self.assertTrue(bool((nz >= split).all() and (nz < slot_hi).all())) def test_translator_takes_the_fused_disposition_on_the_mla_host(self): - """A draft runner bound over the MLA host's entries must translate - through the mamba allocator's full-side v2p, exactly as on the SWA - host. A kind-specific probe regression here silently reverts the - draft to passthrough (raw virtual ids into the views).""" + """A draft runner bound over the MLA host's entries translates through + the mamba allocator's full-side v2p, as on the SWA host; a passthrough + here would write raw virtual ids into the views.""" from sglang.srt.mem_cache.allocator.unified_mamba import ( UnifiedMambaTokenToKVPoolAllocator, ) @@ -489,7 +402,7 @@ class TestFusedMLAHost(unittest.TestCase): ) self.assertTrue(translator.is_translating) self.assertIs( - translator.full_flat_v2p(), alloc.full_attn_allocator.virtual_to_physical + translator.full_v2p_table, alloc.full_attn_allocator.virtual_to_physical ) def test_draft_pool_binds_over_the_mla_host(self): diff --git a/test/registered/unit/mem_cache/test_full_loc_fast_path.py b/test/registered/unit/mem_cache/test_full_loc_fast_path.py index d5097646a6..6efbda76e0 100644 --- a/test/registered/unit/mem_cache/test_full_loc_fast_path.py +++ b/test/registered/unit/mem_cache/test_full_loc_fast_path.py @@ -280,9 +280,9 @@ class TestHybridLinearMLARouting(unittest.TestCase): return pool def test_set_mla_kv_buffer_door_never_translates(self): - """The translate happens exactly once, at ForwardBatch construction - (`rebind_write_loc`); a door that translated again would - double-translate every unified MLA write.""" + """The translate happens exactly once, in the iteration's plan + (`KVLocPlan`); a door that translated again would double-translate + every unified MLA write.""" pool = self._make_bare_pool() loc = torch.tensor([107, 108, 109], dtype=torch.int64) layer = types.SimpleNamespace(layer_id=0) diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index 54df63e1a2..a8f089038e 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -11,29 +11,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""`place_fused_draft`: which host/draft pairs fuse, and where each runner's -layers land. +"""`place_fused_draft` and the target's fused-draft decision: which drafts +fuse into the host's entries, and where each runner's layers land. -A decline keeps a DFLASH or DSPARK draft on its private pool (a DFLASH draft -on a mamba host is refused instead), and refuses an EAGLE draft at boot. A -wrong answer for a draft that keeps its pool silently changes what it binds -and what the boot solve prices. Pinned: +A decline keeps a draft on a private pool (or refuses it at boot); admitting +one that cannot read the fused rows correctly gives silently wrong drafts. +Pinned: - a replicated head under multi-layer EAGLE gets one lane RANGE per runner (one shared region would let the runners clobber each other's KV); - a per-depth head serves one depth per runner and needs one runner per depth; - a draft with SWA or recurrent-state layers of its own, or asymmetric K/V - rows, declines: the fused arm binds one dense pool over the host's full - slots; - - a draft whose attention backend is off the translated MHA rails - declines: it would read the fused rows without the KV-index translator; - - so does a draft under --dcp-size > 1, where each rank's host rows hold - only its share of the tokens the replicated draft reads, and a draft - under HiCache, which builds its host pool off a device pool of its own; - - an explicit draft KV dtype unlike the host's declines; - - the profile divides the draft's heads by attn_tp, as the target does; - - a placement whose runner lane counts do not fill its region is refused; - - the priced entry counts the layers THIS runner owns, not the whole model's. + rows, declines; + - so does a draft whose attention backend is off the translated MHA rails + (it would read the fused rows with virtual ids), a draft under + --dcp-size > 1 (each rank's rows hold only its share of the tokens), an + explicit draft KV dtype unlike the host's, and a draft under HiCache or + host-pool decode retraction (they build the draft's host pool off a + device pool of its own). python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v """ @@ -48,9 +43,6 @@ from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, DraftKVGeometry, DraftKVProfile, - FusedDraftDecision, - FusedDraftPlacement, - draft_kv_profile, place_fused_draft, ) from sglang.srt.runtime_context import get_parallel, override_platform @@ -130,118 +122,6 @@ class TestPlaceFusedDraft(CustomTestCase): ) -class TestDraftKVProfile(CustomTestCase): - def test_heads_are_divided_by_attn_tp(self): - mc = SimpleNamespace( - is_hybrid_swa=True, - is_deepseek_v4_arch=False, - swa_attention_layer_ids=[0], - num_nextn_predict_layers=None, - get_num_kv_heads=lambda tp: max(1, 8 // tp), - head_dim=64, - v_head_dim=32, - ) - with patch("sglang.srt.configs.hybrid_arch.mambaish_config", return_value=None): - profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=2) - self.assertEqual( - profile.full, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32) - ) - self.assertEqual(profile.swa_layer_ids, (0,)) - self.assertEqual(profile.num_depths, 1) - self.assertEqual(profile.num_state_layers, 0) - - def _linear_trunk_mc(self, hf_text_config): - return SimpleNamespace( - is_hybrid_swa=False, - is_deepseek_v4_arch=False, - num_nextn_predict_layers=1, - get_num_kv_heads=lambda tp: 8, - head_dim=64, - v_head_dim=64, - hf_text_config=hf_text_config, - ) - - def test_a_nextn_head_of_a_linear_trunk_owns_no_state(self): - """BUG REGRESSION. A NEXTN head ships inside the trunk checkpoint and - inherits its config CLASS, so `mamba2_cache_params` lists the TRUNK's - state layers while the head is a full-attention block. Counting them - declined a fusable draft to its private pool.""" - mc = self._linear_trunk_mc(SimpleNamespace()) - trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2])) - with patch( - "sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk - ): - profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1) - self.assertEqual(profile.num_state_layers, 0) - - def test_a_conv_chain_head_still_counts_its_state(self): - """The discriminator must not silence a head that really owns state: - only a conv-chain MTP config declares `mtp_local_layer_ids`.""" - mc = self._linear_trunk_mc(SimpleNamespace(mtp_local_layer_ids=[0])) - trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2])) - with patch( - "sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk - ): - profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1) - self.assertEqual(profile.num_state_layers, 3) - - -class TestFusedDraftPlacement(CustomTestCase): - def test_runner_lane_counts_must_fill_the_region(self): - region = DenseDraftRegion( - lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE - ) - with self.assertRaises(AssertionError): - FusedDraftPlacement(region=region, runner_lane_counts=(1,)) - with self.assertRaises(AssertionError): - FusedDraftPlacement(region=region, runner_lane_counts=()) - - -class TestFusedEntryPricing(CustomTestCase): - """The boot solve prices a fused entry through the same spec the pool - factory builds, so the two cannot drift. The factory builds the full - sub-pool from this runner's OWN layer slice; pricing from the whole-model - split over-counts every layer another pipeline rank holds, and the solve - then hands out fewer tokens than fit.""" - - def _configurator(self, *, whole, hybrid_swa, owned=(), span=(0, 0)): - from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator - - cfg = KVCacheConfigurator.__new__(KVCacheConfigurator) - cfg.is_hybrid_swa = hybrid_swa - cfg.layer_info = SimpleNamespace( - full_attention_layer_ids=list(owned), - start_layer=span[0], - end_layer=span[1], - ) - cfg.mambaish_config = SimpleNamespace(full_attention_layer_ids=whole) - cfg.model_config = SimpleNamespace( - full_attention_layer_ids=whole, - head_dim=8, - get_num_kv_heads=lambda *_: 1, - ) - cfg.use_mla_backend = False - cfg.kv_cache_dtype = _DTYPE - return cfg - - def _priced(self, cfg): - region = DenseDraftRegion( - lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE - ) - with get_parallel().override(attn_tp_size=1, attn_dcp_size=1): - return cfg._full_host_spec(region).layer_num - - def test_an_swa_host_prices_this_runners_slice(self): - cfg = self._configurator( - whole=[0, 1, 2, 3], hybrid_swa=True, owned=[0, 1], span=(0, 2) - ) - self.assertEqual(self._priced(cfg), 2) - - def test_a_mamba_host_prices_only_the_layers_in_its_span(self): - cfg = self._configurator(whole=[0, 1, 2, 3], hybrid_swa=False, span=(2, 4)) - self.assertEqual(self._priced(cfg), 2) - - class TestFusedDraftDecision(CustomTestCase): """The target's boot decision over a host whose full sub-pool fuses.""" @@ -252,7 +132,6 @@ class TestFusedDraftDecision(CustomTestCase): draft_kv_dtype=None, host_kv_dtype=_DTYPE, kv_cache_dtype_flag="auto", - attention_arch=None, draft_backend=None, target_backends=("triton", "triton"), dcp_size=1, @@ -286,7 +165,7 @@ class TestFusedDraftDecision(CustomTestCase): head_dim=64, v_head_dim=64, dtype=torch.bfloat16, - attention_arch=attention_arch or AttentionArch.MHA, + attention_arch=AttentionArch.MHA, ), ) memory = SimpleNamespace( @@ -326,15 +205,6 @@ class TestFusedDraftDecision(CustomTestCase): def test_a_stateless_draft_is_placed(self): self.assertIsNotNone(self._decide().placement) - def test_a_non_mha_draft_keeps_the_private_pool(self): - """The region holds dense MHA K/V rows. An MLA draft is kept out by its - own architecture, not by the accident of asymmetric head dims.""" - from sglang.srt.configs.model_config import AttentionArch - - declined = self._decide(attention_arch=AttentionArch.MLA) - self.assertIsNone(declined.placement) - self.assertIn("MLA", declined.declined) - def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self): """A fused draft stores its rows in the host's KV dtype; an explicit draft dtype that differs declines instead of mis-typing those rows, @@ -400,90 +270,5 @@ class TestFusedDraftDecision(CustomTestCase): ) -class TestMambaHostPrivateDraftRefused(CustomTestCase): - """A mamba host's unified buffer takes the whole KV budget, so a private - draft pool on top of it would overcommit; the fused arm is the only - EAGLE-family or DFLASH arm such a host builds. A DSPARK draft that does - not fuse keeps its private pool.""" - - _DECLINED = "the draft's K/V rows are asymmetric" - - def _resolve(self, *, decision, algorithm="EAGLE"): - from sglang.srt.mem_cache import kv_cache_configurator as kvc - from sglang.srt.speculative.spec_info import SpeculativeAlgorithm - - cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator) - cfg.is_draft_worker = False - cfg.spec_algorithm = SpeculativeAlgorithm[algorithm] - with patch.object( - kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision - ): - return cfg._fused_draft_for_mamba_factory() - - def test_a_declined_eagle_draft_is_refused(self): - for algorithm in ("EAGLE", "EAGLE3"): - with self.subTest(algorithm=algorithm): - with self.assertRaisesRegex(ValueError, "rows are asymmetric"): - self._resolve( - decision=FusedDraftDecision(declined=self._DECLINED), - algorithm=algorithm, - ) - - def test_a_declined_dflash_draft_is_refused_and_dspark_keeps_its_pool(self): - """A DFLASH decliner is refused like EAGLE's; a DSPARK decliner keeps - its private pool instead of failing the boot.""" - declined = FusedDraftDecision(declined=self._DECLINED) - with self.assertRaisesRegex(ValueError, "rows are asymmetric"): - self._resolve(decision=declined, algorithm="DFLASH") - self.assertIsNone(self._resolve(decision=declined, algorithm="DSPARK")) - - def test_a_placed_draft_and_other_algorithms_pass(self): - placement = _place(_profile()).placement - for algorithm in ("EAGLE", "DFLASH", "DSPARK"): - with self.subTest(algorithm=algorithm): - self.assertIs( - self._resolve( - decision=FusedDraftDecision(placement=placement), - algorithm=algorithm, - ), - placement, - ) - self.assertIsNone( - self._resolve(decision=FusedDraftDecision(), algorithm="NONE") - ) - - -class TestSWAHostPrivateDraftRefused(CustomTestCase): - """A hybrid-SWA host's boot solve prices a private EAGLE draft at the - target's per-token size, so a draft that does not fuse is refused there - too; the fused arm is the only EAGLE arm such a host builds.""" - - def _resolve(self, *, decision, eagle=True): - from sglang.srt.mem_cache import kv_cache_configurator as kvc - - cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator) - cfg.is_draft_worker = False - cfg.spec_algorithm = SimpleNamespace(is_eagle=lambda: eagle) - with patch.object( - kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision - ): - return cfg._fused_draft_for_swa_factory() - - def test_a_declined_eagle_draft_is_refused(self): - with self.assertRaisesRegex(ValueError, "KV cache dtype"): - self._resolve( - decision=FusedDraftDecision( - declined="the draft's KV cache dtype (bf16) differs from the host's" - ) - ) - - def test_a_placed_draft_and_other_algorithms_pass(self): - placement = _place(_profile()).placement - self.assertIs( - self._resolve(decision=FusedDraftDecision(placement=placement)), placement - ) - self.assertIsNone(self._resolve(decision=FusedDraftDecision(), eagle=False)) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py index b40273a815..4b9a0b7c91 100644 --- a/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py +++ b/test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py @@ -11,20 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Draft-shaped KVCacheConfigurator dispatch under the unified pool. +"""How a draft worker's KVCacheConfigurator binds its KV under the unified pool. -BUG REGRESSION (fast path). The fast path was gated on `req_to_token_pool is -None` alone, but a compact-window DFLASH draft (`--speculative-draft-window-size`) -also passes None — it builds a private req_to_token of its own — so the draft -worker would allocate a SECOND unified byte buffer at boot. A draft-shaped -configurator must fall through to the normal pool build instead. - -DERIVED PROPERTY (binding dispatch). The fused-vs-private draft binding -dispatches on the spec algorithm, not the allocator kind: a non-EAGLE draft on -a unified SWA target takes the private arm, sized by the full sub-allocator's -VIRTUAL id space (`max_slots - 1`). The SWA allocator's `size_full` reports -the static token budget — smaller than the id space — so sizing by it would -put verify-window writes at high virtual ids out of bounds. +With a fused region the draft binds `UnifiedDraftKVPool` over the target's +allocator (a compact-window DFLASH draft with a private `req_to_token` of its +own), refusing a KV dtype unlike the region's. Without one it takes the private +arm, sized by the full sub-allocator's VIRTUAL id space (`max_slots - 1`), not +`size_full`, the smaller static token budget: sizing by that would put +verify-window writes at high virtual ids out of bounds. python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v """ @@ -56,82 +50,7 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -class _ReachedNormalBuild(Exception): - """Sentinel: control flow fell past the unified fast path.""" - - -class TestUnifiedFastPathDraftGate(CustomTestCase): - def _run(self, *, is_draft_worker: bool): - cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) - cfg.is_draft_worker = is_draft_worker - cfg.mambaish_config = object() # would select the mamba unified arm - cfg.is_hybrid_swa = False # not the mamba+SWA tri-pool arm - cfg.model_config = SimpleNamespace(hf_config={}) # not DSV4 - # Every field the fast path reads: a missing one raises while the - # arm's arguments are evaluated, which reads here as "took the fast - # path but called nothing". - sizes = SimpleNamespace( - max_running_requests=8, - max_total_num_tokens=64, - full_max_total_num_tokens=64, - swa_max_total_num_tokens=32, - unified_total_bytes=1 << 20, - ) - taken = [] - with ( - patch.object( - kcc, - "get_memory", - return_value=SimpleNamespace(enable_unified_memory=True), - ), - patch.object( - kcc, - "get_disagg", - return_value=SimpleNamespace(disaggregation_mode="null"), - ), - patch.object( - kcc.KVCacheConfigurator, - "_init_unified_mamba_pools", - lambda self, **kw: taken.append("mamba") or None, - ), - patch.object( - kcc.KVCacheConfigurator, - "_init_unified_swa_pools", - lambda self, **kw: taken.append("swa") or None, - ), - patch.object( - kcc.KVCacheConfigurator, - "_build_req_to_token_pool", - side_effect=_ReachedNormalBuild, - ), - ): - try: - cfg._init_pools( - sizes=sizes, - req_to_token_pool=None, - token_to_kv_pool_allocator=None, - ) - except _ReachedNormalBuild: - return "normal", taken - except (AttributeError, TypeError): - # The stubbed unified arm returned None; anything past the - # fast-path dispatch counts as having taken it. - return "unified", taken - return "unified", taken - - def test_draft_worker_falls_through_to_the_normal_build(self): - path, taken = self._run(is_draft_worker=True) - self.assertEqual(path, "normal") - self.assertEqual(taken, []) - - def test_target_worker_still_takes_the_fast_path(self): - """The guard must narrow to drafts only — a target regression here - silently turns --enable-unified-memory into a no-op.""" - path, taken = self._run(is_draft_worker=False) - self.assertEqual(taken, ["mamba"]) +register_cpu_ci(est_time=5, stage="weekly", runner_config="cpu") class _FakeKVCache: @@ -240,7 +159,6 @@ class TestDraftBindingDispatch(CustomTestCase): alloc, max_total_num_tokens, kv_dtype=torch.bfloat16, - state_layers=(), req_to_token_pool="shared", ): if req_to_token_pool == "shared": @@ -295,9 +213,7 @@ class TestDraftBindingDispatch(CustomTestCase): # The real scan walks the draft nn.Module for RadixAttention layers. patch.object(unified_draft_pool, "draft_kv_layer_ids", return_value=[0]), patch.object( - unified_draft_pool, - "draft_state_layer_classes", - return_value=list(state_layers), + unified_draft_pool, "draft_state_layer_classes", return_value=[] ), patch.object( kcc.KVCacheConfigurator, @@ -325,34 +241,33 @@ class TestDraftBindingDispatch(CustomTestCase): sized = caught.exception.sizes.max_total_num_tokens self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS) - def test_eagle_draft_still_binds_the_fused_pool(self): + def test_a_draft_with_a_region_binds_the_fused_pool(self): + """An EAGLE or DSPARK draft binds the fused pool over the target's + allocator when the target resolved a region; the region-less private + arm above is the fallback.""" alloc = self._swa_allocator(with_draft_region=True) - pools = self._run( - algorithm=SpeculativeAlgorithm.EAGLE3, - alloc=alloc, - max_total_num_tokens=alloc.size_full, - ) - self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) - self.assertIs(pools.token_to_kv_pool_allocator, alloc) + for algorithm in (SpeculativeAlgorithm.EAGLE3, SpeculativeAlgorithm.DSPARK): + pools = self._run( + algorithm=algorithm, + alloc=alloc, + max_total_num_tokens=alloc.size_full, + ) + self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) + self.assertIs(pools.token_to_kv_pool_allocator, alloc) def test_a_draft_kv_dtype_unlike_the_region_refuses_to_bind(self): - """The region stores the target's KV dtype. A draft that resolved its - own -- `auto` picking up an fp8 quant config the target lacks, say -- - must fail loudly rather than read those rows as another dtype.""" - alloc = self._swa_allocator(with_draft_region=True) + """The region stores the target's KV dtype, every fp8 flavor as uint8: + a draft resolving the target's fp8 binds a pool that casts to it, and + one resolving another dtype refuses instead of reading the rows as + that dtype.""" + bf16 = self._swa_allocator(with_draft_region=True) with self.assertRaisesRegex(ValueError, "speculative-draft-kv-cache-dtype"): self._run( algorithm=SpeculativeAlgorithm.EAGLE3, - alloc=alloc, - max_total_num_tokens=alloc.size_full, + alloc=bf16, + max_total_num_tokens=bf16.size_full, kv_dtype=torch.float8_e4m3fn, ) - - def test_an_fp8_region_binds_an_fp8_draft_and_refuses_another_fp8(self): - """Every fp8 flavor is stored as uint8, so the bind compares KV dtypes: - a draft resolving the target's fp8 binds a pool that casts to that fp8, - and one resolving a different fp8 refuses instead of reading e4m3 bytes - as e5m2.""" alloc = self._swa_allocator( with_draft_region=True, kv_dtype=torch.float8_e4m3fn ) @@ -372,30 +287,6 @@ class TestDraftBindingDispatch(CustomTestCase): kv_dtype=torch.float8_e5m2, ) - def test_a_draft_with_state_layers_refuses_to_bind(self): - """The target places a draft from its config; the built model is the - ground truth. Recurrent layers would run with no state pool.""" - alloc = self._swa_allocator(with_draft_region=True) - with self.assertRaisesRegex(ValueError, "MambaMixer2"): - self._run( - algorithm=SpeculativeAlgorithm.EAGLE3, - alloc=alloc, - max_total_num_tokens=alloc.size_full, - state_layers=["MambaMixer2"], - ) - - def test_dspark_draft_with_a_region_binds_the_fused_pool(self): - """DSPARK's draft KV fuses when the target resolved a region (its - block rows are indexed by the same token->page identity); the - region-less private arm above remains the automatic fallback.""" - alloc = self._swa_allocator(with_draft_region=True) - pools = self._run( - algorithm=SpeculativeAlgorithm.DSPARK, - alloc=alloc, - max_total_num_tokens=alloc.size_full, - ) - self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) - def test_compact_dflash_draft_fuses_with_a_private_req_table(self): """Compact-window DFLASH passes req_to_token_pool=None (it keeps a private table narrowing WHICH pages the draft reads) while the KV @@ -411,22 +302,6 @@ class TestDraftBindingDispatch(CustomTestCase): self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool) self.assertEqual(pools.req_to_token_pool.kind, "private-compact") - def test_eagle_draft_without_a_placement_falls_back_to_the_private_arm(self): - """Target boot declines a placement for legitimate configurations (a - draft with SWA layers of its own, asymmetric rows), so a - placement-less EAGLE draft binds the private pool -- sized by the id - space like any other private draft -- instead of failing the boot.""" - alloc = self._swa_allocator(with_draft_region=False) - id_space = alloc.full_attn_allocator.max_slots - 1 - with self.assertRaises(_CapturedSizes) as caught: - self._run( - algorithm=SpeculativeAlgorithm.EAGLE3, - alloc=alloc, - max_total_num_tokens=alloc.size_full, - ) - sized = caught.exception.sizes.max_total_num_tokens - self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_index_translator.py b/test/registered/unit/mem_cache/test_kv_index_translator.py index da57ee82b6..b660569587 100644 --- a/test/registered/unit/mem_cache/test_kv_index_translator.py +++ b/test/registered/unit/mem_cache/test_kv_index_translator.py @@ -11,7 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""KVIndexTranslator -- the read-path id translator. +"""KVIndexTranslator -- what a runner reads and writes an iteration's plan +through: the plan's read table and write ids per disposition, the readers +(`read_source`, `pack_read_stream`, `copy_page_table`), and the boot-time +dispositions. CPU-only: these exercise the builder's pure-torch reference path, not the Triton kernel. @@ -32,7 +35,8 @@ from test_multi_ended_allocator import _FakeKVCache, _FakeUnifiedSWAKVPool from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, ) -from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator, KVReadTables +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool from sglang.srt.state_capturer.base import BaseTopkCapturer @@ -113,12 +117,26 @@ def _reference_table(req_to_token, req_pool_indices, seq_lens, v2p, ps, width): return out +def _publish(): + # The code under test reads its config from the bags. + reset_context() + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + +def _plan(src, rows, seq_lens, *, write_virtual=None, read_extent=0): + return src.plan( + req_pool_indices=rows, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens.clone(), + write_virtual=write_virtual, + read_extent=read_extent, + ) + + class TestPassthrough(unittest.TestCase): def setUp(self): - # The code under test reads its config from the bags. - reset_context() + _publish() self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") def test_non_unified_returns_same_objects(self): """Strict passthrough: no copy, no branch. Any tensor op on the @@ -134,18 +152,15 @@ class TestPassthrough(unittest.TestCase): self.assertFalse(src.is_translating) self.assertEqual(src.capture_token_capacity(17), 18) rows = torch.tensor([2, 0]) - view = src.build_index_table( - req_pool_indices=rows, seq_lens=torch.tensor([5, 3]) - ) + loc = torch.tensor([1, 2, 3]) + plan = _plan(src, rows, torch.tensor([5, 3]), write_virtual=loc) + self.assertIs(plan.write_ids(src), loc) + view = src.read_source(plan, req_pool_indices=rows, bs=2) self.assertIs(view.ids, req_to_token) self.assertIs(view.row_ids, rows) self.assertEqual(view.row_stride, req_to_token.stride(0)) self.assertEqual(view.entry_page_size, 1) self.assertFalse(view.is_translated) - self.assertIsNone(view.sliding_window_ids) - # And the translate surface is the identity, not a wrapped copy. - t = torch.tensor([1, 2, 3]) - self.assertIs(src.translate_full_attn_ids(t), t) def _alloc_and_fill(allocator, ps, lens): @@ -167,10 +182,8 @@ def _alloc_and_fill(allocator, ps, lens): class TestReadTableBuild(unittest.TestCase): def setUp(self): - # The code under test reads its config from the bags. - reset_context() + _publish() self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") def test_read_table_matches_reference(self): """Both read tables must equal the independent per-element derivation @@ -183,71 +196,27 @@ class TestReadTableBuild(unittest.TestCase): ) src = _make_source(allocator, req_to_token, ps) self.assertTrue(src.is_translating) - width = 6 - view = src.build_index_table( - req_pool_indices=rows, seq_lens=seq_lens, max_pages=width - ) + plan = _plan(src, rows, seq_lens) + view = src.read_table(plan) + width = view.ids.shape[1] + self.assertEqual(width, 5) # the widest row's pages self.assertTrue(view.is_translated) self.assertEqual(view.entry_page_size, ps) self.assertTrue( torch.equal(view.row_ids, torch.arange(3, dtype=torch.int64)) ) - want_full = _reference_table( - req_to_token, - rows, - seq_lens, - allocator.full_v2p_page_table, - ps, - width, - ) - want_swa = _reference_table( - req_to_token, - rows, - seq_lens, - allocator.swa_v2p_page_table, - ps, - width, - ) - self.assertTrue( - torch.equal(view.ids, want_full), - f"full read table off-formula (ps={ps})", - ) - self.assertTrue( - torch.equal(view.sliding_window_ids, want_swa), - f"swa read table off-formula (ps={ps})", - ) - - def test_packed_stream_equals_the_rectangle_it_replaces(self): - """The two builders must agree element for element: - packed[indptr[b] + p] == ids[b, p // ps] * ps + p % ps.""" - for ps in (1, 4): - allocator = _build_composite(ps) - req_to_token, rows, seq_lens = _alloc_and_fill( - allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1] - ) - src = _make_source(allocator, req_to_token, ps) - view = src.build_index_table( - req_pool_indices=rows, seq_lens=seq_lens, max_pages=6 - ) - indptr = torch.zeros(len(seq_lens) + 1, dtype=torch.int32) - indptr[1:] = torch.cumsum(seq_lens, dim=0) - total = int(indptr[-1]) - packed = torch.zeros(total, dtype=torch.int32) - translated = src.fill_packed_read_stream( - req_pool_indices=rows, - seq_lens=seq_lens, - indptr=indptr, - total_tokens=total, - out=packed, - ) - self.assertTrue(translated) - for b, n in enumerate(seq_lens.tolist()): - for pos in range(n): - self.assertEqual( - int(packed[int(indptr[b]) + pos]), - int(view.ids[b, pos // ps]) * ps + pos % ps, - f"packed stream off the page table (ps={ps}, b={b}, {pos=})", - ) + for table, v2p, side in ( + (view.ids, allocator.full_v2p_page_table, "full"), + ( + src.read_table(plan, kind=IdSpaceKind.SLIDING_WINDOW).ids, + allocator.swa_v2p_page_table, + "swa", + ), + ): + want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width) + self.assertTrue( + torch.equal(table, want), f"{side} read table off-formula (ps={ps})" + ) def test_sink_routing(self): """Dead lanes, -1 slots inside the live prefix, and tombstoned v2p @@ -265,64 +234,112 @@ class TestReadTableBuild(unittest.TestCase): allocator.full_v2p_page_table[tomb_page] = -1 allocator.swa_v2p_page_table[tomb_page] = -1 src = _make_source(allocator, req_to_token, ps) - view = src.build_index_table( - req_pool_indices=rows, seq_lens=seq_lens, max_pages=4 - ) - for table in (view.ids, view.sliding_window_ids): + plan = _plan(src, rows, seq_lens) + for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW): + table = src.read_table(plan, kind=kind).ids self.assertTrue(bool((table >= 0).all())) self.assertTrue(bool((table[1] == 0).all()), "dead lane not sunk") self.assertEqual(int(table[0, 1]), 0, "-1 slot not sunk") self.assertEqual(int(table[2, 0]), 0, "tombstone not sunk") -class TestBuildInto(unittest.TestCase): - """fill_read_table fills a backend-owned padded block table's live prefix with - FULL-side read-table entries -- the trtllm_mla / flashmla consumption route - (their rows ARE the read table's rows).""" +class TestReadExtent(unittest.TestCase): + """A forward that reads past its lengths (a verify's draft tail) plans its + table that far: `seq_lens + read_extent`, capped at the request row.""" def setUp(self): - # The code under test reads its config from the bags. - reset_context() + _publish() self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") - def test_prefix_filled_tail_sentinel_preserved_width_capped(self): - """The -1 tail sentinel belongs to the backend, and a table padded - WIDER than the req_to_token page span (trtllm's LCM alignment) must be - capped rather than trip the builder's width assert.""" + def test_table_covers_the_extent(self): + window = 3 + for ps in (1, 4): + allocator = _build_composite(ps) + req_to_token, rows, live = _alloc_and_fill( + allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1] + ) + src = _make_source(allocator, req_to_token, ps) + view = src.read_table(_plan(src, rows, live, read_extent=window)) + width = -(-(int(live.max()) + window) // ps) + self.assertEqual(tuple(view.ids.shape), (rows.numel(), width)) + want = _reference_table( + req_to_token, + rows, + live + window, + allocator.full_v2p_page_table, + ps, + width, + ) + self.assertTrue(torch.equal(view.ids, want), f"ps={ps}") + + def test_width_never_exceeds_the_request_row(self): + """Near the context limit the host bound plus the extent can pass the + end of the `req_to_token` row; the table stops at the row instead of + tripping the read-table width assert.""" + window = 6 + allocator = _build_composite(1) + req_to_token, rows, live = _alloc_and_fill(allocator, 1, lens=[6, 1]) + row_pages = req_to_token.shape[1] + src = _make_source(allocator, req_to_token, 1) + plan = src.plan( + req_pool_indices=rows, + seq_lens=live, + seq_lens_cpu=live + 2 * window, + write_virtual=None, + read_extent=window, + ) + self.assertGreater(int(live.max()) + 3 * window, row_pages) + self.assertEqual(src.read_table(plan).ids.shape[1], row_pages) + + +class TestCopyPageTable(unittest.TestCase): + """`copy_page_table`: a captured graph's own table takes the plan's rows, + padded lanes reading the sink; columns past the plan's width keep their + values (the kernels bound their reads by their own lengths).""" + + def setUp(self): + _publish() + self.addCleanup(reset_context) + + def test_rows_copied_tail_kept_padded_lanes_sunk(self): ps = 4 allocator = _build_composite(ps) - lens = [5, 2 * ps + 1, 1] + lens = [5, 2 * ps + 1] req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens) src = _make_source(allocator, req_to_token, ps) - self.assertTrue(src.is_translating) + plan = _plan(src, rows, seq_lens) + width = src.read_table(plan).ids.shape[1] + # Wider than the req_to_token span and one lane more than the batch. + out = torch.full((3, req_to_token.shape[1] // ps + 3), 7, dtype=torch.int32) + out_swa = torch.full_like(out, 7) + src.copy_page_table(plan, out=out) + src.copy_page_table(plan, out=out_swa, kind=IdSpaceKind.SLIDING_WINDOW) + for v2p, got in ( + (allocator.full_v2p_page_table, out), + (allocator.swa_v2p_page_table, out_swa), + ): + want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width) + self.assertTrue(torch.equal(got[:2, :width], want)) + self.assertTrue(bool((got[2, :width] == 0).all()), "padded lane") + self.assertTrue(bool((got[:, width:] == 7).all()), "tail clobbered") - width_pages = req_to_token.shape[1] // ps + 3 # wider than the span - out = torch.full((len(lens), width_pages), -1, dtype=torch.int32) - src.fill_read_table(out=out, req_pool_indices=rows, seq_lens=seq_lens) - - want = _reference_table( - req_to_token, - rows, - seq_lens, - allocator.full_v2p_page_table, - ps, - width_pages, - ) - for b, n in enumerate(lens): - n_pages = -(-n // ps) - self.assertTrue( - torch.equal(out[b, :n_pages], want[b, :n_pages]), - f"row {b} live prefix off-formula", - ) - self.assertTrue( - bool((out[b, n_pages:] == -1).all()), - f"row {b} tail sentinel clobbered", - ) + def test_row_ids_not_reallocated_across_plans(self): + """`row_ids` is one arange sized from the request pool and sliced per + table; a per-table `torch.arange` would cost an allocation and a launch + on every replay prep.""" + allocator = _build_composite(1) + req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3]) + src = _make_source(allocator, req_to_token, 1) + first = src.read_table(_plan(src, rows[:2], seq_lens[:2])) + second = src.read_table(_plan(src, rows, seq_lens)) + self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr()) + self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV))) + self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV))) + self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0]) def test_passthrough_source_refuses(self): """Callers dispatch on `reads_are_translated`; a passthrough source has - no v2p to build from and must fail loud, not fill garbage.""" + no plan table to copy and must fail loud, not fill garbage.""" src = KVIndexTranslator( req_to_token=torch.zeros((2, 4), dtype=torch.int64), token_to_kv_pool_allocator=SimpleNamespace(), @@ -330,12 +347,9 @@ class TestBuildInto(unittest.TestCase): page_size=1, device=_DEV, ) + plan = _plan(src, torch.tensor([0]), torch.tensor([1])) with self.assertRaises(AssertionError): - src.fill_read_table( - out=torch.zeros((1, 4), dtype=torch.int32), - req_pool_indices=torch.tensor([0]), - seq_lens=torch.tensor([1]), - ) + src.copy_page_table(plan, out=torch.zeros((1, 4), dtype=torch.int32)) class TestPoolOwnership(unittest.TestCase): @@ -422,81 +436,26 @@ class TestPoolOwnership(unittest.TestCase): device=_DEV, ) rows = torch.tensor([1, 0], dtype=torch.int32, device=_DEV) - view = src.build_index_table( - req_pool_indices=rows, - seq_lens=torch.tensor([3, 2], dtype=torch.int32, device=_DEV), + loc = torch.tensor([5, 6], dtype=torch.int64, device=_DEV) + plan = _plan( + src, + rows, + torch.tensor([3, 2], dtype=torch.int32, device=_DEV), + write_virtual=loc, ) - # Read table: the EXACT objects a static-pool backend reads today. + # Writes and reads: the EXACT objects a static-pool backend uses. + self.assertIs(plan.write_ids(src), loc) + view = src.read_source(plan, req_pool_indices=rows, bs=2) self.assertIs(view.ids, req_to_token) self.assertIs(view.row_ids, rows) self.assertFalse(view.is_translated) - # And the token-level surface is the identity, same guard. - t = torch.tensor([5, 6], dtype=torch.int64, device=_DEV) - self.assertIs(src.translate_full_attn_ids(t), t) - - -class TestCaptureContract(unittest.TestCase): - def setUp(self): - # The code under test reads its config from the bags. - reset_context() - self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") - - def test_caller_owned_table_is_returned_whole_and_filled_prefix_only(self): - ps = 4 - allocator = _build_composite(ps) - req_to_token = torch.full((4, 16 * ps), -1, dtype=torch.int64) - v = allocator.alloc(2 * ps) - req_to_token[1, : 2 * ps] = v - src = _make_source(allocator, req_to_token, ps) - - # A page-table consumer owns its buffers; entry 0 is the reserved sink - # in every id space, so zeros are what an unfilled column must read as. - cap = torch.zeros((4, 8), dtype=torch.int32, device=_DEV) - cap_swa = torch.zeros((4, 8), dtype=torch.int32, device=_DEV) - tables = KVReadTables(full=cap, sliding_window=cap_swa) - - # Poison everything, then refresh a 1-row batch: ONLY its live prefix - # may change -- stale tails and other rows are the fa3 contract. - cap.fill_(7) - cap_swa.fill_(7) - view = src.build_index_table( - req_pool_indices=torch.tensor([1]), - seq_lens=torch.tensor([2 * ps]), - into=tables, - ) - self.assertIs(view.ids, cap, "the caller's table comes back WHOLE") - want = allocator.full_v2p_page_table[req_to_token[1, ::ps][:2] // ps] - self.assertTrue(torch.equal(cap[0, :2], want.to(torch.int32))) - self.assertTrue(bool((cap[0, 2:] == 7).all()), "stale tail was cleared") - self.assertTrue(bool((cap[1:] == 7).all()), "rows beyond bs were touched") - - def test_row_ids_not_reallocated_across_builds(self): - """`row_ids` is one arange sized from the request pool and sliced per - build; a per-build `torch.arange` would be correct but costs an - allocation and a launch on every replay prep.""" - allocator = _build_composite(1) - req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3]) - src = _make_source(allocator, req_to_token, 1) - self.assertTrue(src.is_translating) - first = src.build_index_table( - req_pool_indices=rows[:2], seq_lens=seq_lens[:2], max_pages=4 - ) - second = src.build_index_table( - req_pool_indices=rows, seq_lens=seq_lens, max_pages=4 - ) - self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr()) - self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV))) - self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV))) - # Sized to bound any batch the request pool can hold. - self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0]) class _FakeForwardBatch: - """Weakref-able stand-in (SimpleNamespace is not) carrying the fields - `index_table_for_batch` and `rebind_write_loc` read. `seq_lens_sum` - defaults to the real sum: it is the signal that the CPU mirror is live, - and a real ForwardBatch always carries it (None only when gpu_only).""" + """Stand-in carrying the fields a plan of its own (`own_plan`) and the + capturers read. `seq_lens_sum` defaults to the real sum: it is the signal + that the CPU mirror is live, and a real ForwardBatch always carries it + (None only when gpu_only).""" def __init__( self, @@ -520,138 +479,14 @@ class _FakeForwardBatch: ) -class TestWidenedTable(unittest.TestCase): - """`widened_index_table`: the whole-sequence verify view, `seq_len_delta` - columns past every row's live prefix.""" +class TestWriteIds(CustomTestCase): + """A plan translates its write window once, into a fresh tensor, and + derives the sliding-window ids from the virtual window -- pads and + tombstones landing on the sink.""" def setUp(self): - # The code under test reads its config from the bags. - reset_context() + _publish() self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") - - def _check(self, view, req_to_token, rows, lens, allocator, ps, width): - self.assertEqual(tuple(view.ids.shape), (rows.numel(), width)) - want = _reference_table( - req_to_token, rows, lens, allocator.full_v2p_page_table, ps, width - ) - self.assertTrue(torch.equal(view.ids, want), f"ps={ps}") - - def test_width_comes_from_the_live_prefix(self): - """A DSPARK verify batch carries `seq_lens_cpu` already expanded by the - window and the live lens on `spec_info.live_seq_lens_cpu`; widening the - expanded lens would count the window twice.""" - window = 3 - for ps in (1, 4): - allocator = _build_composite(ps) - req_to_token, rows, live = _alloc_and_fill( - allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1] - ) - src = _make_source(allocator, req_to_token, ps) - fb = _FakeForwardBatch( - req_pool_indices=rows, - seq_lens=live, - seq_lens_cpu=live + window, - spec_info=SimpleNamespace(live_seq_lens_cpu=live), - ) - view = src.widened_index_table(fb, seq_len_delta=window) - width = -(-(int(live.max()) + window) // ps) - self._check(view, req_to_token, rows, live + window, allocator, ps, width) - - def test_width_never_exceeds_the_request_row(self): - """Near the context limit the widened host bound can pass the end of - the `req_to_token` row; the table stops at the row instead of tripping - the read-table width assert.""" - window = 6 - ps = 1 - allocator = _build_composite(ps) - req_to_token, rows, live = _alloc_and_fill(allocator, ps, lens=[6, 1]) - row_pages = req_to_token.shape[1] // ps - src = _make_source(allocator, req_to_token, ps) - # No live lens on the spec info: the expanded mirror is all there is. - fb = _FakeForwardBatch( - req_pool_indices=rows, seq_lens=live, seq_lens_cpu=live + window - ) - self.assertGreater(int(live.max()) + 2 * window, row_pages) - view = src.widened_index_table(fb, seq_len_delta=window) - self._check(view, req_to_token, rows, live + window, allocator, ps, row_pages) - - -class TestViewMemo(unittest.TestCase): - """The eager view is memoized ON THE SOURCE in a single slot keyed by batch - identity, so per-batch state stays out of the ForwardBatch while one - metadata build's many consumers still share one table build.""" - - def setUp(self): - # The code under test reads its config from the bags. - reset_context() - self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") - - def _fb(self, allocator, ps, lens): - req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens) - fb = _FakeForwardBatch( - req_pool_indices=rows, - seq_lens=seq_lens, - seq_lens_cpu=seq_lens, - ) - return fb, req_to_token - - def test_same_batch_returns_the_memoized_view(self): - ps = 1 - allocator = _build_composite(ps) - fb, req_to_token = self._fb(allocator, ps, lens=[3, 2]) - src = _make_source(allocator, req_to_token, ps) - v1 = src.index_table_for_batch(fb) - v2 = src.index_table_for_batch(fb) - self.assertIs(v1, v2) - - def test_next_batch_replaces_the_single_slot(self): - ps = 1 - allocator = _build_composite(ps) - fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2]) - src = _make_source(allocator, req_to_token, ps) - v1 = src.index_table_for_batch(fb1) - fb2 = _FakeForwardBatch( - req_pool_indices=fb1.req_pool_indices, - seq_lens=fb1.seq_lens, - seq_lens_cpu=fb1.seq_lens_cpu, - ) - v2 = src.index_table_for_batch(fb2) - self.assertIsNot(v1, v2) - # Single slot: fb1 no longer matches and rebuilds. - v1_again = src.index_table_for_batch(fb1) - self.assertIsNot(v1_again, v1) - - def test_dead_batch_never_matches(self): - """A garbage-collected batch's slot must not serve a later batch: the - weakref key goes dead and the build runs fresh.""" - import gc - - ps = 1 - allocator = _build_composite(ps) - fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2]) - src = _make_source(allocator, req_to_token, ps) - v1 = src.index_table_for_batch(fb1) - del fb1 - gc.collect() - fb2, _ = self._fb(allocator, ps, lens=[2]) - v2 = src.index_table_for_batch(fb2) - self.assertIsNot(v2, v1) - self.assertEqual(v2.ids.shape[0], 1) - - -class TestWriteLoc(CustomTestCase): - """The two-phase write contract: `rebind_write_loc` rebinds the full side - once at ForwardBatch construction, and the sliding-window write loc derives - POINTWISE from the full-side values -- pads, slices, and fresh copies - included -- with no handover and no stored per-forward state.""" - - def setUp(self): - # The code under test reads its config from the bags. - reset_context() - self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") def _built(self, ps=1, n=4): allocator = _build_composite(ps) @@ -662,20 +497,18 @@ class TestWriteLoc(CustomTestCase): want_swa = allocator.translate_loc_from_full_to_swa(virt) return src, allocator, rows, seq_lens, virt, want_full, want_swa - def _field(self, src, rows, seq_lens, kernel_loc): - return src.sliding_window_write_loc_for(kernel_loc) - - def test_rebind_translates_full_side_only(self): + def test_own_plan_translates_full_side_only(self): for ps in (1, 4): src, _, _, _, virt, want_full, _ = self._built(ps=ps, n=3 * ps) keep = virt.clone() fb = _FakeForwardBatch(out_cache_loc=virt) - src.rebind_write_loc(fb) - # Full side: rebound to a FRESH physical tensor; the - # ScheduleBatch's aliased virtual tensor is untouched. + src.bind_own_plan(fb) + # Full side: a FRESH physical tensor; the ScheduleBatch's aliased + # virtual tensor is untouched and mirrored for its bookkeeping. self.assertIsNot(fb.out_cache_loc, virt) self.assertTrue(torch.equal(fb.out_cache_loc, want_full)) self.assertTrue(torch.equal(virt, keep)) + self.assertIs(fb.out_cache_loc_virtual, virt) def test_capture_capacity_covers_dcp_widened_mamba_ids(self): """A capturer must store the highest virtual IDs issued under DCP.""" @@ -755,7 +588,7 @@ class TestWriteLoc(CustomTestCase): req_pool = SimpleNamespace(req_to_token=virt.clone()[None, :]) fb = _FakeForwardBatch(out_cache_loc=virt.clone()) if translating: - src.rebind_write_loc(fb) + src.bind_own_plan(fb) fb.out_cache_loc = torch.cat( [fb.out_cache_loc, virt.new_zeros(2)] ) @@ -795,35 +628,36 @@ class TestWriteLoc(CustomTestCase): ) ) - def test_swa_write_loc_round_trips_from_full_side(self): - """Derived property: `field(full(t)) == swa(t)` for any virtual run t, - across page sizes.""" + def test_swa_ids_come_from_the_virtual_ids(self): for ps in (1, 4, 64): - src, _, rows, seq_lens, _, want_full, want_swa = self._built( - ps=ps, n=3 * ps + src, _, rows, seq_lens, virt, _, want_swa = self._built(ps=ps, n=3 * ps) + plan = _plan(src, rows, seq_lens, write_virtual=virt) + self.assertTrue( + torch.equal( + plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW), want_swa + ) ) - got = self._field(src, rows, seq_lens, want_full) - self.assertTrue(torch.equal(got, want_swa)) - def test_pad_lanes_derive_to_sink(self): - """The DP pad appends zeros, and physical 0 is the reserved - padding slot in every id space, so pad lanes derive to swa slot 0 with - no `num_live` bookkeeping.""" - src, _, rows, seq_lens, _, want_full, want_swa = self._built(n=3) - padded = torch.cat([want_full, want_full.new_zeros(2)]) - got = self._field(src, rows, seq_lens, padded) + def test_pad_lanes_read_the_sink(self): + """The DP pad appends zeros, and id 0 is the reserved padding slot in + every id space, so pad lanes land on swa slot 0.""" + src, _, rows, seq_lens, virt, _, want_swa = self._built(n=3) + padded = torch.cat([virt, virt.new_zeros(2)]) + plan = _plan(src, rows, seq_lens, write_virtual=padded) + got = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW) self.assertTrue(torch.equal(got[:3], want_swa)) self.assertTrue(bool((got[3:] == 0).all()), "pad lanes must land on slot 0") - def test_tombstoned_swa_page_clamps_to_sink(self): - src, allocator, rows, seq_lens, virt, want_full, _ = self._built(ps=1, n=2) + def test_tombstoned_swa_page_reads_the_sink(self): + src, allocator, rows, seq_lens, virt, _, _ = self._built(ps=1, n=2) allocator.swa_v2p_page_table[int(virt[0])] = -1 - got = self._field(src, rows, seq_lens, want_full[:1]) - self.assertEqual(int(got[0]), 0) + plan = _plan(src, rows, seq_lens, write_virtual=virt[:1]) + swa_ids = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW) + self.assertEqual(int(swa_ids[0]), 0) def test_static_swa_pool_derives_via_pool_translate(self): - """Static SWA pools: the field is the pool's own legacy full->swa - translate, computed at the same build; the rebind stays a no-op.""" + """Static SWA pools: the swa ids are the pool's own full->swa + translate; the write ids stay the batch's own.""" pool = SWAKVPool.__new__(SWAKVPool) pool.full_to_swa_index_mapping = torch.arange(10, dtype=torch.int64) pool.translate_loc_from_full_to_swa = lambda t: t + 100 @@ -836,14 +670,18 @@ class TestWriteLoc(CustomTestCase): ) loc = torch.tensor([5, 6], dtype=torch.int64) fb = _FakeForwardBatch(out_cache_loc=loc) - src.rebind_write_loc(fb) - self.assertIs(fb.out_cache_loc, loc, "disabled rebind must be a no-op") - self.assertTrue(torch.equal(src.sliding_window_write_loc_for(loc), loc + 100)) + src.bind_own_plan(fb) + self.assertIs(fb.out_cache_loc, loc, "a static pool's ids are its own") + self.assertTrue( + torch.equal(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), loc + 100) + ) def test_no_loc_or_no_swa_side_yields_none(self): # Unified swa composite, but there is no write loc this forward. src, _, rows, seq_lens, _, _, _ = self._built(n=2) - self.assertIsNone(src.sliding_window_write_loc_for(None)) + self.assertIsNone( + _plan(src, rows, seq_lens).write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW) + ) # Passthrough on a non-SWA pool: a loc is given, but there is no swa # id space to derive into. plain = KVIndexTranslator( @@ -853,23 +691,13 @@ class TestWriteLoc(CustomTestCase): page_size=1, device=_DEV, ) + loc = torch.tensor([3], dtype=torch.int64) self.assertIsNone( - plain.sliding_window_write_loc_for(torch.tensor([3], dtype=torch.int64)) + _plan( + plain, torch.tensor([0]), torch.tensor([1]), write_virtual=loc + ).write_ids(plain, kind=IdSpaceKind.SLIDING_WINDOW) ) - def test_rebind_retires_the_view_memo(self): - ps = 1 - allocator = _build_composite(ps) - req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=[3, 2]) - src = _make_source(allocator, req_to_token, ps) - fb = _FakeForwardBatch( - req_pool_indices=rows, seq_lens=seq_lens, seq_lens_cpu=seq_lens - ) - v1 = src.index_table_for_batch(fb) - src.rebind_write_loc(_FakeForwardBatch(out_cache_loc=None)) - v2 = src.index_table_for_batch(fb) - self.assertIsNot(v2, v1, "rebind starts the next forward: stale views die") - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py index 2056a50d5a..2e1d1a7c67 100644 --- a/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py +++ b/test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py @@ -11,36 +11,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""KVIndexTranslator's three id-space dispositions with a fused draft region. +"""KVIndexTranslator with a fused draft region. -1. TARGET runner (allocator's own kvcache): translates to physical ids. -2. FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to this allocator): the - SAME translate -- same pages, same v2p table, same physical ids (the draft - parts sit inside the host entry). A dense draft pool routes no window - layers, so it has no swa id space: the window index table falls back to - the one dense table and `sliding_window_write_loc_for()` answers None (a - separate-swa assumption here crashed the read side and left the write side - with nothing to derive). -3. PRIVATE-POOL draft (DSPARK/DFLASH shape: target's allocator, own - virtual-indexed buffer): strict passthrough — translating such a runner - would address a slot-count buffer with dense ids (OOB both directions), - so the no-op is load-bearing, not a default. +A FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to the target's allocator) +translates exactly as the target does -- same v2p table, same physical ids, +the draft parts sitting inside the host entries -- and, routing no window +layers, has no sliding-window write ids. A PRIVATE-POOL draft (own +virtual-indexed buffer) stays a strict passthrough. The draft writers outside +the draft's forward (DFLASH/DSPARK's target-hidden writes, the multi-step +draft decode's kv indices) read the same ids. python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v """ -import ast -import pathlib import unittest from types import SimpleNamespace import torch -from sglang.srt.mem_cache import kv_index_translator from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, ) from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, FusedDraftPlacement, @@ -51,7 +44,7 @@ from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu") +register_cpu_ci(est_time=6, suite="base-a-test-cpu") _DEV = "cpu" _PS = 2 @@ -150,163 +143,32 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): self.addCleanup(reset_context) publish(ServerArgs(model_path="dummy"), role="tokenizer") - def test_target_runner_translates_to_physical_ids(self): - _, allocator, kvcache, _ = _build() - src = _source(allocator, kvcache) - self.assertTrue(src.is_translating) - v = allocator.alloc(2 * _PS) - self.assertIsNotNone(v) - fb = SimpleNamespace(out_cache_loc=v) - src.rebind_write_loc(fb) - self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v))) - def test_fused_draft_runner_translates_to_the_same_physical_ids(self): _, allocator, _, draft_pool = _build() src = _source(allocator, draft_pool) self.assertTrue(src.is_translating) - self.assertIsNone(src._swa_v2p_table) # single space: no swa id space + # A dense draft pool: no sliding-window sub-pool. + self.assertIsNone(src.space(IdSpaceKind.SLIDING_WINDOW)) - # rebind_write_loc translates to the HOST's physical ids: the draft - # parts live inside the same slots. + # Its own plan translates to the HOST's physical ids: the draft parts + # live inside the same slots. v = allocator.alloc(2 * _PS) self.assertIsNotNone(v) fb = SimpleNamespace(out_cache_loc=v) - src.rebind_write_loc(fb) + src.bind_own_plan(fb) self.assertIsNot(fb.out_cache_loc, v) fa = allocator.full_attn_allocator expected = torch.clamp_min(fa.virtual_to_physical[v // _PS] * _PS + v % _PS, 0) torch.testing.assert_close(fb.out_cache_loc, expected, rtol=0, atol=0) self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v))) # A dense draft pool routes no window layers: no swa write loc. - self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc)) - - def test_dense_fused_draft_has_no_window_write_loc(self): - """`sliding_window_write_loc_for()` on a dense fused-draft batch answers - None: the pool routes no window layers, so there is nothing to - derive. The hybrid-SWA TARGET on the same allocator keeps a genuinely - derived swa loc, so the None can never paper over a real swa id - space.""" - _, allocator, kvcache, draft_pool = _build() - src = _source(allocator, draft_pool) - v = allocator.alloc(_PS) - self.assertIsNotNone(v) - fb = SimpleNamespace(out_cache_loc=v) - src.rebind_write_loc(fb) - self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc)) - # Target contrast: a real swa side derives a DIFFERENT loc. - tgt = _source(allocator, kvcache) - tv = allocator.alloc(_PS) - self.assertIsNotNone(tv) - tfb = SimpleNamespace(out_cache_loc=tv) - tgt.rebind_write_loc(tfb) - tswa = tgt.sliding_window_write_loc_for(tfb.out_cache_loc) - self.assertIsNotNone(tswa) - self.assertIsNot(tswa, tfb.out_cache_loc) - self.assertTrue(torch.equal(tswa, allocator.translate_loc_from_full_to_swa(tv))) - - def test_private_pool_draft_stays_a_strict_passthrough(self): - _, allocator, _, _ = _build() - private_draft_pool = _FakeKVCache(64) # own buffer, not the allocator's - src = _source(allocator, private_draft_pool) - self.assertFalse(src.is_translating) - v = torch.arange(2 * _PS, dtype=torch.int64) - fb = SimpleNamespace(out_cache_loc=v) - src.rebind_write_loc(fb) - self.assertIs(fb.out_cache_loc, v) # untouched, not even a copy - - def test_foreign_allocator_draft_pool_is_not_enabled(self): - # A UnifiedDraftKVPool bound to a DIFFERENT allocator must not enable - # against this one (identity, not type, decides). - pool, allocator, kvcache, _ = _build() - _, other_alloc, _, other_draft = _build() - src = _source(allocator, other_draft) - self.assertFalse(src.is_translating) - - def test_seq_len_delta_matches_widened_lens(self): - """``seq_len_delta=k`` must be byte-identical to building with - ``seq_lens + k`` -- the two spellings of the whole-sequence verify - widening. A kernel applying the delta to the page count but not the - loads (or vice versa) silently truncates the verify tail.""" - _, allocator, kvcache, _ = _build() - v = allocator.alloc(6 * _PS) - self.assertIsNotNone(v) - rt = torch.zeros((2, 16), dtype=torch.int32) - rt[0, : v.numel()] = v.to(torch.int32) - src = KVIndexTranslator( - req_to_token=rt, - token_to_kv_pool_allocator=allocator, - token_to_kv_pool=kvcache, - page_size=_PS, - device=_DEV, - ) - rpi = torch.tensor([0], dtype=torch.int64) - seq = torch.tensor([3], dtype=torch.int64) - delta = 2 * _PS + 1 - max_pages = -(-(3 + delta) // _PS) - widened = src.build_index_table( - req_pool_indices=rpi, - seq_lens=seq, - max_pages=max_pages, - seq_len_delta=delta, - ) - by_lens = src.build_index_table( - req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages - ) - torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0) - # The delta genuinely widened: entries exist past the unwidened prefix. - plain_pages = -(-3 // _PS) - self.assertTrue(bool((widened.ids[0, plain_pages:] > 0).any())) - - def test_widened_index_table_matches_widened_lens(self): - """`widened_index_table` (the verify entry point) must equal the - widened-lens build over the SAME batch: derive max_pages from the - widened max and forward the delta. Deriving max_pages from the - un-widened lens silently truncates the verify tail's pages.""" - _, allocator, kvcache, _ = _build() - v = allocator.alloc(6 * _PS) - self.assertIsNotNone(v) - rt = torch.zeros((2, 16), dtype=torch.int32) - rt[0, : v.numel()] = v.to(torch.int32) - src = KVIndexTranslator( - req_to_token=rt, - token_to_kv_pool_allocator=allocator, - token_to_kv_pool=kvcache, - page_size=_PS, - device=_DEV, - ) - rpi = torch.tensor([0], dtype=torch.int64) - seq = torch.tensor([3], dtype=torch.int64) - delta = 2 * _PS + 1 - fb = SimpleNamespace( - req_pool_indices=rpi, - seq_lens=seq, - seq_lens_cpu=seq.cpu(), - # `seq_lens_sum` is the liveness signal for seq_lens_cpu: it is a - # non-None but STALE slice on a gpu_only batch, so the build only - # trusts it when the sum is present. Without the field the stand-in - # batch falls back to the full req_to_token width. - seq_lens_sum=int(seq.sum()), - out_cache_loc=None, - spec_info=None, - ) - widened = src.widened_index_table(fb, seq_len_delta=delta) - max_pages = -(-(3 + delta) // _PS) - by_lens = src.build_index_table( - req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages - ) - self.assertEqual(widened.ids.shape, by_lens.ids.shape) - torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0) + self.assertIsNone(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW)) def test_target_hidden_writers_write_physical_ids(self): - """BUG REGRESSION. DFLASH and DSPARK do not compute their draft KV from - the draft's own forward -- they PROJECT the target's hidden states and - write them straight into the draft pool, at locs read off the target's - req_to_token (VIRTUAL). Under fusion the draft pool takes the target's - physical ids, so an untranslated write lands at the wrong row: the - draft attends over the wrong KV and accept length collapses to 1.0 - with no crash. Each writer must hand the pool translated ids and leave - the caller's locs virtual (the compact req_to_token rebuild re-reads - them).""" + """DFLASH and DSPARK project the target's hidden states straight into + the draft pool. Under fusion that pool takes the target's physical ids: + both writers take them from the iteration's plan in the draft pool's + space, never the virtual ids (a wrong row, with no crash).""" from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2 from sglang.srt.speculative.dspark_components.dspark_kv_inject import ( TargetHiddenKvInjector, @@ -317,11 +179,15 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): is_translating = True - def translate_full_attn_ids(self, ids, *, out=None): - return ids + 100 - virtual = torch.tensor([4, 5, 6, 9], dtype=torch.int64) physical = virtual + 100 + draft_translator = _Shift() + # The iteration's plan: physical ids for the fused draft's translator. + plan = SimpleNamespace( + write_ids=lambda reader, cols=None: ( + physical.clone() if reader is draft_translator else virtual.clone() + ) + ) hidden = torch.randn(4, 8) positions = torch.arange(4) commit_lens = torch.tensor([2, 1], dtype=torch.int32) @@ -348,7 +214,7 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): worker = SimpleNamespace( model_runner=SimpleNamespace(device=torch.device(_DEV)), draft_model_runner=SimpleNamespace( - kv_index_translator=_Shift(), token_to_kv_pool=_DflashPool() + kv_index_translator=draft_translator, token_to_kv_pool=_DflashPool() ), draft_model=SimpleNamespace( project_target_hidden=lambda h: h, @@ -365,30 +231,29 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): ) # Per-token writes (prefill). - cache_loc = virtual.clone() + cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan) DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc( worker, target_hidden=hidden, cache_loc=cache_loc, positions=positions ) self.assertTrue(writes["loc"].physical) torch.testing.assert_close(writes["loc"].loc, physical, rtol=0, atol=0) - torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0) # Prefix-valid writes (post-verify), from the 2-D view of the same ids. - cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2) + cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan) DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc( worker, target_hidden=hidden, cache_loc=cache_loc, positions=positions, - cache_loc_2d=cache_loc_2d, + cache_loc_2d=cache_loc.view(2, 2), commit_lens=commit_lens, ) torch.testing.assert_close( writes["loc_2d"], physical.view(2, 2), rtol=0, atol=0 ) - torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0) - # DSPARK's injector, on an MHA draft pool. + # DSPARK's injector, on an MHA draft pool: the plan's ids for the + # draft's translator, written as they are. injected = {} def write_target_hidden_kv(**kwargs): @@ -397,50 +262,47 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): injector = TargetHiddenKvInjector( draft_model=SimpleNamespace(write_target_hidden_kv=write_target_hidden_kv), draft_model_runner=SimpleNamespace( - kv_index_translator=_Shift(), token_to_kv_pool=SimpleNamespace() + kv_index_translator=draft_translator, token_to_kv_pool=SimpleNamespace() ), model_runner=SimpleNamespace(device=torch.device(_DEV)), device=torch.device(_DEV), verify_num_draft_tokens=2, block_pos_offsets=torch.arange(2), ) - cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2) + cache_loc = injector.ids_for(plan) injector.inject_target_hidden( target_hidden=hidden, cache_loc=cache_loc, positions=positions, - cache_loc_2d=cache_loc_2d, + cache_loc_2d=cache_loc.view(2, 2), commit_lens=commit_lens, ) torch.testing.assert_close(injected["cache_loc"], physical, rtol=0, atol=0) torch.testing.assert_close( injected["cache_loc_2d"], physical.view(2, 2), rtol=0, atol=0 ) - torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0) - torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0) - def test_full_flat_v2p_per_disposition(self): - """The flat-translate accessor must hand a kernel exactly what + def test_full_v2p_table_per_disposition(self): + """The v2p accessor must hand a DCP table builder exactly what `translate_kv_loc` would use: the shared v2p table on the target and on the fused draft, and None on a pass-through runner (a translated pass-through would address a private buffer with the target's ids).""" _, allocator, kvcache, draft_pool = _build() target = _source(allocator, kvcache) - self.assertIs(target.full_flat_v2p(), allocator.full_v2p_page_table) + self.assertIs(target.full_v2p_table, allocator.full_v2p_page_table) draft = _source(allocator, draft_pool) - self.assertIs(draft.full_flat_v2p(), allocator.full_v2p_page_table) + self.assertIs(draft.full_v2p_table, allocator.full_v2p_page_table) passthrough = _source(allocator, _FakeKVCache(64)) - self.assertIsNone(passthrough.full_flat_v2p()) + self.assertIsNone(passthrough.full_v2p_table) - def test_multi_step_containers_pass_the_v2p_table_to_the_kernel(self): - """Every `generate_draft_decode_kv_indices` launch must thread the - runner's v2p table and a matching TRANSLATE flag. A launch - without them emits raw req_to_token values, which under the unified - pool are VIRTUAL ids the fused draft pool cannot address — the exact - silent-garbage bug this series fixed.""" + def test_multi_step_containers_read_the_plans_table(self): + """Every `generate_draft_decode_kv_indices` launch must gather from + `read_source` with its entry granularity. A launch over raw + req_to_token emits VIRTUAL ids, which the fused draft pool cannot + address -- silent garbage drafts.""" import pathlib import re @@ -454,112 +316,20 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase): if launches: launching[path.name] = ( launches, - len(re.findall(r"TRANSLATE=v2p is not None", text)), - "full_flat_v2p" in text, + len(re.findall(r"ENTRY_PAGE_SIZE=src\.entry_page_size", text)), + "kv_index_translator.read_source(" in text, ) self.assertGreaterEqual( len(launching), 4, f"launch sites disappeared: {sorted(launching)}" ) - for name, (launches, translated, has_accessor) in launching.items(): + for name, (launches, granular, has_source) in launching.items(): self.assertEqual( launches, - translated, - f"{name}: {launches} kernel launch(es) but only {translated} " - "carry TRANSLATE=v2p is not None", + granular, + f"{name}: {launches} kernel launch(es) but only {granular} " + "pass the read table's entry granularity", ) - self.assertTrue( - has_accessor, - f"{name} launches the kernel without full_flat_v2p", - ) - - -class TestDispositionBranchesAgree(unittest.TestCase): - """Every `__init__` disposition must assign the SAME attribute set. - - BUG REGRESSION. `KVIndexTranslator.__init__` picks a disposition -- - translating (unified target or fused draft) or passthrough -- and later - methods read attributes off `self` unconditionally. When upstream adds an - attribute it naturally adds it to the branches IT knows about; a branch - added here is silently left short, and nothing fails until the missing - attribute is read at RUNTIME. That is exactly how the fused-draft branch - lost `_translate_write_full` and `defer_read_translate` across a rebase: - py_compile passes (the attribute is only ever read, never declared), the - undefined-NAME check passes (it is an attribute, not a bare name), and the - target and passthrough paths both work -- only a fused-draft forward raises - `AttributeError: 'KVIndexTranslator' object has no attribute - '_translate_write_full'`. - - Comparing the branches against EACH OTHER needs no list to maintain: a new - attribute is covered the moment any one branch sets it. - - python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v - """ - - def setUp(self): - # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived - # parallel width that only exists once a config is published. - reset_context() - self.addCleanup(reset_context) - publish(ServerArgs(model_path="dummy"), role="tokenizer") - - def _init_branch_assignments(self): - """{branch index: {attr names it assigns}} for __init__'s if/elif/else.""" - src = pathlib.Path(kv_index_translator.__file__).read_text() - tree = ast.parse(src) - cls = next( - n - for n in ast.walk(tree) - if isinstance(n, ast.ClassDef) and n.name == "KVIndexTranslator" - ) - init = next( - n - for n in cls.body - if isinstance(n, ast.FunctionDef) and n.name == "__init__" - ) - - # The disposition chain is the `if` whose body assigns the id-space - # fields; find it by the attribute every disposition must set. - def assigns(stmts): - out = set() - for st in stmts: - for node in ast.walk(st): - if isinstance(node, ast.Assign): - for t in node.targets: - if ( - isinstance(t, ast.Attribute) - and isinstance(t.value, ast.Name) - and t.value.id == "self" - ): - out.add(t.attr) - return out - - for node in init.body: - if not isinstance(node, ast.If): - continue - branches, cur = [], node - while True: - branches.append(assigns(cur.body)) - if len(cur.orelse) == 1 and isinstance(cur.orelse[0], ast.If): - cur = cur.orelse[0] - continue - if cur.orelse: - branches.append(assigns(cur.orelse)) - break - if len(branches) >= 2 and all("_full_v2p_table" in b for b in branches): - return branches - self.fail("could not locate the disposition if/elif/else in __init__") - - def test_every_disposition_assigns_the_same_attributes(self): - branches = self._init_branch_assignments() - union = set().union(*branches) - missing = {i: sorted(union - b) for i, b in enumerate(branches) if union - b} - self.assertEqual( - missing, - {}, - "a KVIndexTranslator.__init__ disposition does not assign every " - "attribute its siblings do; the branch raises AttributeError only " - "when that path is taken at runtime: " + repr(missing), - ) + self.assertTrue(has_source, f"{name} launches without read_source") if __name__ == "__main__": diff --git a/test/registered/unit/mem_cache/test_kv_loc_plan.py b/test/registered/unit/mem_cache/test_kv_loc_plan.py new file mode 100644 index 0000000000..e093a10336 --- /dev/null +++ b/test/registered/unit/mem_cache/test_kv_loc_plan.py @@ -0,0 +1,745 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`KVLocPlan`: one iteration's KV ids, translated once. + +- the target and a fused draft get the same physical write ids, a + pass-through reader (a private draft pool, a static pool) the virtual + window, which is never mutated; `bind` and a split batch's `cols_slice` + pick the same tokens in both spaces; +- a forward whose `seq_lens` do not yet count the window it writes (a + verify, a speculative draft decode, a draft extend) reads that window past + them; +- one iteration's forwards translate the window once per sub-pool and build + a sub-pool's read table at most once, only when a table reader asks; the + table covers `seq_lens + read_extent`, padded lanes reading the sink; +- only readers of the plan's own rows read its table; a pass-through reader + gathers `req_to_token`, and a draft with its own `req_to_token` plans its + own reads over the shared write ids (`reads_from`); +- a runner's own write buffer and a replayed graph's padded buffer are used + as they are, the sliding-window ids naming the sink past the plan's. + + python -m pytest test/registered/unit/mem_cache/test_kv_loc_plan.py -v +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import msgspec +import torch + +from sglang.srt.mem_cache import kv_index_translator +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWATokenToKVPoolAllocator, +) +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, TokenSpan +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) +from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool +from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + +_DEV = "cpu" +_PS = 2 +_FULL = IdSpaceKind.FULL +_SWA = IdSpaceKind.SLIDING_WINDOW + + +def _reference(req_to_token, rows, lens, v2p, width): + """Independent derivation of a read table: each row's pages over `lens`, + through `v2p`, the sink past them.""" + out = torch.zeros((rows.numel(), width), dtype=torch.int32) + for b, row in enumerate(rows.tolist()): + for c in range(min(-(-int(lens[b]) // _PS), width)): + out[b, c] = max(int(v2p[int(req_to_token[row, c * _PS]) // _PS]), 0) + return out + + +class _HostStreamGather: + """`create_flashinfer_kv_indices_triton` on the host, for these CPU tests: + lane ``b``'s ids from row ``row_ids[b]`` of the source, packed at + ``indptr[b]``.""" + + def __getitem__(self, grid): + return self + + def __call__( + self, + ids, + row_ids, + seq_lens, + indptr, + kv_start_idx, + out, + row_stride, + ENTRY_PAGE_SIZE=1, + token_mapping=None, + ): + assert row_stride == ids.stride(0) + for b in range(int(seq_lens.numel())): + start = 0 if kv_start_idx is None else int(kv_start_idx[b]) + pos = torch.arange(start, start + int(seq_lens[b])) + entry = ids[int(row_ids[b]), pos // ENTRY_PAGE_SIZE].to(torch.int64) + tok = entry * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE + if token_mapping is not None: + tok = token_mapping[tok] + lo = int(indptr[b]) + out[lo : lo + int(seq_lens[b])] = tok.to(out.dtype) + + +class _FakeKVCache: + def __init__(self, max_slots): + self.buf = torch.full((max_slots,), -1, dtype=torch.int64) + self.allocator = None + + def attach_allocator(self, allocator): + self.allocator = allocator + + +class _FakeUnifiedSWAKVPool: + def __init__(self, shared_pool): + self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full")) + self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa")) + self.full_to_swa_index_mapping = None + + def attach_allocators(self, *, full_allocator, swa_allocator): + self._full_allocator = full_allocator + self._swa_allocator = swa_allocator + + +class TestKVLocPlan(unittest.TestCase): + def setUp(self): + # `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived + # parallel width that only exists once a config is published. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + full_spec = MHASubPoolSpec( + name="full", + layer_num=2, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="down", + draft_region=DenseDraftRegion( + lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16 + ), + ) + swa_spec = MHASubPoolSpec( + name="swa", + layer_num=1, + head_num=2, + head_dim=4, + store_dtype=torch.bfloat16, + grow_direction="up", + ) + n_full, n_swa = 32, 16 + pool = UnifiedKVPool( + total_bytes=n_full * full_spec.entry_bytes() + + n_swa * swa_spec.entry_bytes(), + sub_pool_specs=[full_spec, swa_spec], + device=_DEV, + enable_memory_saver=False, + page_size=_PS, + fused_draft=FusedDraftPlacement( + region=full_spec.draft_region, runner_lane_counts=(1,) + ), + ) + kvcache = _FakeUnifiedSWAKVPool(pool) + self.allocator = UnifiedSWATokenToKVPoolAllocator( + unified_buffer=pool, + kvcache=kvcache, + device=_DEV, + full_max_total_num_tokens=n_full, + swa_max_total_num_tokens=n_swa, + page_size=_PS, + need_sort=False, + forward_stream=None, + ) + draft_pool = UnifiedDraftKVPool( + unified_buffer=pool, + host_sub_pool_name="full", + host_allocator=self.allocator, + layer_lanes={0: 0}, + page_size=_PS, + ) + # Every runner of one server shares the target's req_to_token. + self.req_to_token = torch.zeros((4, 16), dtype=torch.int32) + self.draft_pool = draft_pool + self.target = self._translator(kvcache) + self.fused_draft = self._translator(draft_pool) + self.private_draft = self._translator(_FakeKVCache(64)) + + # Two requests, two pages each; the second page of each is this + # iteration's write window (one token per request). + self.rpi = torch.tensor([1, 2], dtype=torch.int64) + for row in (1, 2): + ids = self.allocator.alloc(2 * _PS) + self.assertIsNotNone(ids) + self.req_to_token[row, : 2 * _PS] = ids.to(torch.int32) + self.seq_lens = torch.tensor([3, 3], dtype=torch.int64) + self.window = self.req_to_token[self.rpi, 2:4].to(torch.int64).reshape(-1) + + def _translator(self, pool): + return KVIndexTranslator( + req_to_token=self.req_to_token, + token_to_kv_pool_allocator=self.allocator, + token_to_kv_pool=pool, + page_size=_PS, + device=_DEV, + ) + + def _plan(self, source=None, read_extent=1): + return (source or self.target).plan( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=self.seq_lens.clone(), + write_virtual=self.window, + read_extent=read_extent, + ) + + def test_readers_get_ids_in_their_own_space(self): + window = self.window.clone() + plan = self._plan() + physical = self.allocator.translate_write_loc(window) + self.assertTrue(torch.equal(plan.write_ids(self.target), physical)) + # A fused draft shares the target's pages: the same tensor, untranslated. + self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical) + # A private draft pool indexes virtual ids. + self.assertIs(plan.write_ids(self.private_draft), self.window) + self.assertTrue(torch.equal(self.window, window)) # never mutated + + def test_cols_slice_names_a_split_batch_s_tokens(self): + """Two-batch overlap splits a forward's tokens; each half writes its + own tokens of the window, whatever the forward's own columns were.""" + plan = self._plan() + bs = int(self.rpi.numel()) + everything = plan.cols_slice(None, slice(1, None)) + self.assertEqual(everything, TokenSpan(1, self.window.numel())) + self.assertEqual(plan.cols_slice(everything, slice(1, 3)), TokenSpan(2, 4)) + # A static plan's halves, and a translating one's, both read views. + static = self._plan(self.private_draft) + for p, reader in ((plan, self.target), (static, self.private_draft)): + half = p.cols_slice(None, slice(0, 2)) + self.assertIsInstance(half, TokenSpan) + whole = p.write_ids(reader) + ids = p.write_ids(reader, cols=half) + self.assertTrue(torch.equal(ids, whole[:2])) + self.assertEqual( + ids.untyped_storage().data_ptr(), whole.untyped_storage().data_ptr() + ) + first = plan.cols_slice(slice(0, 1), slice(None)) + width = self.window.numel() // bs + self.assertTrue(torch.equal(first, torch.arange(bs) * width)) + ragged = torch.tensor([3, -1, 0]) + self.assertTrue(torch.equal(plan.cols_slice(ragged, slice(1, 3)), ragged[1:])) + for cols, tokens in ( + (None, slice(1, None)), + (everything, slice(1, 3)), + (slice(0, 1), slice(None)), + ): + self.assertTrue( + torch.equal( + plan.write_ids(self.target, cols=plan.cols_slice(cols, tokens)), + plan.write_ids(self.target, cols=cols)[tokens], + ) + ) + + def test_one_iteration_translates_once(self): + writes, builds = [], [] + real_write = self.allocator.translate_write_loc + real_build = kv_index_translator.build_kv_read_table + + swa_writes = [] + real_swa_write = self.allocator.translate_loc_from_full_to_swa + + def counting_write(ids, *a, **kw): + writes.append(ids) + return real_write(ids, *a, **kw) + + def counting_swa_write(ids): + swa_writes.append(ids) + return real_swa_write(ids) + + def counting_build(**kwargs): + builds.append(kwargs["v2p"]) + return real_build(**kwargs) + + with patch.object(kv_index_translator, "build_kv_read_table", counting_build): + # The spaces hold the allocator's translates from when the + # translator was built; count through them. The fused draft's full + # space is the target's, so it counts there too. + for reader, kind, write in ( + (self.target, _FULL, counting_write), + (self.fused_draft, _FULL, counting_write), + (self.target, _SWA, counting_swa_write), + ): + reader._spaces[kind] = msgspec.structs.replace( + reader.space(kind), write=write + ) + plan = self._plan(read_extent=2) + draft, verify, extend = (SimpleNamespace() for _ in range(3)) + plan.bind(draft, self.fused_draft, cols=slice(0, 1)) + plan.bind(verify, self.target) + plan.bind(extend, self.fused_draft) + bs = int(self.rpi.numel()) + for reader in (self.target, self.fused_draft): + # What a packed stream gathers from, and a captured table. + reader.read_source(plan, req_pool_indices=self.rpi, bs=bs + 1) + reader.read_table(plan) + reader.copy_page_table( + plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32) + ) + # The target's sliding-window layers, written and read twice. + for _ in range(2): + self.target.write_ids(verify, _SWA) + self.target.read_table(plan, kind=_SWA) + self.target.copy_page_table( + plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32), kind=_SWA + ) + self.assertEqual(len(writes), 1) + self.assertEqual(len(swa_writes), 1) + # The full and the sliding-window id space, one build each. + self.assertEqual(len(builds), 2) + self.assertTrue(torch.equal(verify.out_cache_loc, plan.write_physical)) + self.assertIs(extend.out_cache_loc, verify.out_cache_loc) + + def test_a_verify_reads_the_window_it_writes(self): + def own_plan(mode, spec_info): + return self.target.own_plan( + SimpleNamespace( + forward_mode=mode, + spec_info=spec_info, + batch_size=int(self.rpi.numel()), + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=self.seq_lens.clone(), + out_cache_loc=self.window, + ) + ) + + width = self.window.numel() // self.rpi.numel() + # A verify's rows read up to `draft_token_num` past their lengths, more + # than the window when the verify is ragged. + spec = SimpleNamespace(draft_token_num=width + 1) + self.assertEqual( + own_plan(ForwardMode.TARGET_VERIFY, spec).read_extent, width + 1 + ) + self.assertEqual(own_plan(ForwardMode.DECODE, spec).read_extent, width) + # A draft extend's lengths take its window after its batch is built. + self.assertEqual(own_plan(ForwardMode.DRAFT_EXTEND_V2, spec).read_extent, width) + # Their lengths already count what an extend or a plain decode writes. + self.assertEqual(own_plan(ForwardMode.EXTEND, spec).read_extent, 0) + self.assertEqual(own_plan(ForwardMode.DECODE, None).read_extent, 0) + + def test_a_stream_only_iteration_builds_no_table(self): + """A plan builds a table only when a table reader asks for it: stream + readers -- one runner, or a target and a fused draft over the same + rows -- each translate in their own gather. Once the table is built, + a stream packs the same ids from it.""" + calls = {"packed": 0, "table": 0} + + def counting(name, real): + def wrapped(**kwargs): + calls[name] += 1 + return real(**kwargs) + + return wrapped + + # A runner of rows no other translator here reads. + self.req_to_token = self.req_to_token.clone() + solo = self._translator(self.allocator.get_kvcache()) + bs = int(self.rpi.numel()) + lens = torch.tensor([3, 2], dtype=torch.int64) + indptr = torch.tensor([0, 3, 5], dtype=torch.int32) + + def pack(translator, plan, out, kind=_FULL): + return translator.pack_read_stream( + plan, + req_pool_indices=self.rpi, + seq_lens=lens, + indptr=indptr, + out=out, + kind=kind, + ) + + with ( + patch.object( + kv_index_translator, + "build_kv_read_table_packed", + counting("packed", kv_index_translator.build_kv_read_table_packed), + ), + patch.object( + kv_index_translator, + "build_kv_read_table", + counting("table", kv_index_translator.build_kv_read_table), + ), + patch.object( + kv_index_translator, + "create_flashinfer_kv_indices_triton", + _HostStreamGather(), + ), + ): + plan = self._plan(solo, read_extent=1) + out = torch.full((6,), -7, dtype=torch.int32) + self.assertTrue(pack(solo, plan, out)) + self.assertEqual(calls, {"packed": 1, "table": 0}) + self.assertFalse(plan.has_read_table()) + for b in range(bs): + row = self.req_to_token[int(self.rpi[b]), : int(lens[b])] + want = self.allocator.translate_write_loc(row.to(torch.int64)) + got = out[int(indptr[b]) : int(indptr[b + 1])].to(torch.int64) + self.assertTrue(torch.equal(got, want)) + self.assertEqual(int(out[5]), -7) # past the stream: untouched + + # A caller holding a padded graph buffer of request rows: the + # lengths size the batch, the extra lanes are left out. + padded = torch.cat([self.rpi, torch.zeros(5, dtype=torch.int64)]) + again = torch.full((6,), -7, dtype=torch.int32) + solo.pack_read_stream( + self._plan(solo, read_extent=1), + req_pool_indices=padded, + seq_lens=lens, + indptr=indptr, + out=again, + ) + self.assertTrue(torch.equal(again, out)) + self.assertEqual(calls, {"packed": 2, "table": 0}) + + # Once a table reader has the table built, a stream packs from it: + # nothing gathered again, the same ids. + plan.read_table() + after = torch.full((6,), -7, dtype=torch.int32) + pack(solo, plan, after) + self.assertEqual(calls, {"packed": 2, "table": 1}) + self.assertTrue(torch.equal(after, out)) + # Nothing asked for the sliding-window table: its stream comes + # alone. + pack(solo, plan, torch.empty(6, dtype=torch.int32), kind=_SWA) + self.assertEqual(calls, {"packed": 3, "table": 1}) + self.assertFalse(plan.has_read_table(_SWA)) + + # Another stream reader of the same rows (a fused draft): still no + # table -- each translates its own gather. + peer = self._translator(self.draft_pool) + for seq_lens_cpu in (self.seq_lens.clone(), None): + both = solo.plan( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=seq_lens_cpu, + write_virtual=self.window, + read_extent=1, + ) + pack(solo, both, torch.empty(6, dtype=torch.int32)) + pack(peer, both, torch.empty(6, dtype=torch.int32)) + # The draft's row gather gets the virtual rows and the page + # table, translated in its own kernel. + src = peer.read_source(both, req_pool_indices=self.rpi, bs=bs) + self.assertTrue(src.is_translated) + self.assertIs(src.ids, self.req_to_token) + self.assertIs(src.v2p, solo.space(_FULL).read_v2p) + self.assertFalse(both.has_read_table()) + self.assertEqual(calls, {"packed": 7, "table": 1}) + + def test_a_captured_first_reader_holds_the_table(self): + """When the plan's first reader is a captured table, the table is built + in that buffer -- no table of the plan's own, no copy -- and serves the + plan's other readers; the same buffer again is left alone, and another + captured reader copies it rather than building again; an eager reader + of the batch takes the batch's rows of it.""" + builds = [] + real = kv_index_translator.build_kv_read_table + + def counting(**kwargs): + builds.append(kwargs["out"]) + return real(**kwargs) + + bs = int(self.rpi.numel()) + captured = torch.full((bs + 1, 8), 7, dtype=torch.int32) + with patch.object(kv_index_translator, "build_kv_read_table", counting): + plan = self._plan(read_extent=1) + self.target.copy_page_table(plan, out=captured) + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0].data_ptr(), captured.data_ptr()) + table = plan.read_table(rows=bs + 1) + self.assertEqual(table.ids.data_ptr(), captured.data_ptr()) + self.assertEqual(table.row_stride, captured.stride(0)) + # An eager reader of the batch (fa3, trtllm_mha) gets the batch's + # rows of that table: its kernels size the batch by them. + eager = self.target.read_table(plan, rows=bs) + self.assertEqual(eager.ids.shape[0], bs) + self.assertEqual(eager.row_ids.numel(), bs) + self.assertEqual(eager.ids.data_ptr(), captured.data_ptr()) + width = table.ids.shape[1] + reference = _reference( + self.req_to_token, + self.rpi, + self.seq_lens + 1, + self.allocator.full_v2p_page_table, + width, + ) + self.assertTrue(torch.equal(captured[:bs, :width], reference)) + self.assertEqual(int(captured[bs, :width].abs().sum()), 0) # padded lane + self.assertTrue(bool((captured[:, width:] == 7).all())) # past the width + self.target.copy_page_table(plan, out=captured) + other = torch.full_like(captured, 5) + self.target.copy_page_table(plan, out=other) + self.assertEqual(len(builds), 1) + self.assertTrue(torch.equal(other[:, :width], captured[:, :width])) + + def test_a_captured_table_bounds_a_build_without_host_lengths(self): + """Without host lengths, a captured first reader narrower than a row + has the table built in its own buffer, to its width.""" + builds = [] + real = kv_index_translator.build_kv_read_table + + def counting(**kwargs): + builds.append(kwargs["out"]) + return real(**kwargs) + + bs = int(self.rpi.numel()) + row_pages = -(-self.req_to_token.shape[1] // _PS) + captured = torch.full((bs + 1, row_pages // 2), 7, dtype=torch.int32) + with patch.object(kv_index_translator, "build_kv_read_table", counting): + plan = self.target.plan( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=None, + write_virtual=self.window, + read_extent=1, + ) + self.target.copy_page_table(plan, out=captured) + self.assertEqual(len(builds), 1) + self.assertEqual(builds[0].data_ptr(), captured.data_ptr()) + self.assertEqual(builds[0].shape[1], captured.shape[1]) + reference = _reference( + self.req_to_token, + self.rpi, + self.seq_lens + 1, + self.allocator.full_v2p_page_table, + captured.shape[1], + ) + self.assertTrue(torch.equal(captured[:bs], reference)) + self.assertEqual(int(captured[bs].abs().sum()), 0) # padded lane + + def test_read_table_is_built_once_and_padded_by_copy(self): + builds = [] + real = kv_index_translator.build_kv_read_table + + def counting(**kwargs): + builds.append(kwargs["v2p"]) + return real(**kwargs) + + with patch.object(kv_index_translator, "build_kv_read_table", counting): + plan = self._plan(read_extent=1) + table = plan.read_table() + self.assertIs(plan.read_table(), table) + # Only the space that was read. + self.assertEqual(len(builds), 1) + swa = plan.read_table(kind=_SWA) + self.assertEqual(len(builds), 2) + padded = plan.read_table(rows=4) + padded_swa = plan.read_table(kind=_SWA, rows=4) + self.assertEqual(len(builds), 2) # copied, not built again + self.assertTrue(torch.equal(padded.ids[:2], table.ids)) + self.assertEqual(int(padded.ids[2:].abs().sum()), 0) # the sink + self.assertTrue(torch.equal(padded_swa.ids[:2], swa.ids)) + self.assertEqual(int(padded_swa.ids[2:].abs().sum()), 0) + # The rows cover `seq_lens + read_extent`. + reference = _reference( + self.req_to_token, + self.rpi, + self.seq_lens + 1, + self.allocator.full_v2p_page_table, + table.ids.shape[1], + ) + self.assertTrue(torch.equal(table.ids, reference)) + + def test_only_readers_of_its_rows_read_its_table(self): + plan = self._plan() + self.assertTrue(plan.is_read_by(self.target)) + self.assertTrue(plan.is_read_by(self.fused_draft)) + self.assertFalse(plan.is_read_by(self.private_draft)) + compact = KVIndexTranslator( + req_to_token=self.req_to_token.clone(), + token_to_kv_pool_allocator=self.allocator, + token_to_kv_pool=self.draft_pool, + page_size=_PS, + device=_DEV, + ) + self.assertFalse(plan.is_read_by(compact)) + # Before a table reader asks, the fused draft's row gather translates + # in its own kernel; after, it reads the plan's table. + src = self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2) + self.assertIs(src.v2p, self.target.space(_FULL).read_v2p) + self.assertFalse(plan.has_read_table()) + table = plan.read_table(rows=2) + self.assertIs( + self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2), table + ) + passthrough = self.private_draft.read_source( + plan, req_pool_indices=self.rpi, bs=2 + ) + self.assertFalse(passthrough.is_translated) + self.assertIs(passthrough.ids, self.req_to_token) + + def test_a_compact_draft_reads_its_own_rows_over_the_shared_writes(self): + plan = self._plan() + compact_rows = self.req_to_token.clone() + compact = KVIndexTranslator( + req_to_token=compact_rows, + token_to_kv_pool_allocator=self.allocator, + token_to_kv_pool=self.draft_pool, + page_size=_PS, + device=_DEV, + ) + lens = torch.tensor([2, 1], dtype=torch.int64) + derived = plan.reads_from( + compact, seq_lens=lens, seq_lens_cpu=lens.clone(), read_extent=1 + ) + # The writes are the plan's, not translated again. + self.assertIs(derived.write_physical, plan.write_physical) + self.assertIs(derived.write_ids(compact), plan.write_physical) + self.assertTrue(derived.is_read_by(compact)) + compact_rows[self.rpi, :2] = compact_rows[self.rpi, 2:4] + reference = _reference( + compact_rows, + self.rpi, + lens + 1, + self.allocator.full_v2p_page_table, + derived.read_table().ids.shape[1], + ) + self.assertTrue(torch.equal(derived.read_table().ids, reference)) + + def test_bind_sets_the_write_fields(self): + """`bind` gives a batch the columns it writes, the same ones in both + spaces, with the virtual mirror only when its ids were translated.""" + plan = self._plan() + bs = int(self.rpi.numel()) + first = slice(0, 1) + fused = SimpleNamespace() + plan.bind(fused, self.fused_draft, cols=first) + self.assertIs(fused.kv_loc_plan, plan) + self.assertEqual(fused.kv_loc_cols, first) + self.assertTrue( + torch.equal( + fused.out_cache_loc, + plan.write_physical.view(bs, -1)[:, first].reshape(-1), + ) + ) + self.assertTrue( + torch.equal( + fused.out_cache_loc_virtual, + self.window.view(bs, -1)[:, first].reshape(-1), + ) + ) + self.assertTrue(fused.out_cache_loc_is_physical) + private = SimpleNamespace() + plan.bind(private, self.private_draft) + self.assertIs(private.out_cache_loc, self.window) + self.assertIsNone(private.out_cache_loc_virtual) + + def test_an_npu_dcp_target_writes_rank_local_slots(self): + """On NPU with DCP a target writes its rank's slots, while the + scheduler, `req_to_token` and the replicated draft pool keep + allocator-global ones: a plan's window stays global whichever runner + built it (a target prefill handing its plan to the draft, a draft + handing its window to the verify), the target binds it rank-local, + and the draft binds it global.""" + parallel = SimpleNamespace( + dcp_enabled=True, dcp_size=2, dcp_rank=1, attn_dcp_size=1 + ) + with ( + patch.object(kv_index_translator, "_is_npu", True), + patch.object(kv_index_translator, "get_parallel", return_value=parallel), + ): + target, draft = ( + KVIndexTranslator( + req_to_token=self.req_to_token, + token_to_kv_pool_allocator=self.allocator, + token_to_kv_pool=_FakeKVCache(64), + page_size=4, + device=_DEV, + is_draft_worker=is_draft_worker, + ) + for is_draft_worker in (False, True) + ) + global_ids = torch.arange(8, 16) + # Rank 1 of 2 owns every other 4-slot page, compacted. + local_ids = torch.tensor([-1, -1, -1, -1, 4, 5, 6, 7]) + for source in (target, draft): + plan = source.plan( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=self.seq_lens.clone(), + write_virtual=global_ids, + ) + target_batch, draft_batch = SimpleNamespace(), SimpleNamespace() + plan.bind(target_batch, target) + plan.bind(draft_batch, draft) + self.assertTrue(torch.equal(target_batch.out_cache_loc, local_ids)) + self.assertTrue(torch.equal(draft_batch.out_cache_loc, global_ids)) + self.assertIs(plan.write_virtual, global_ids) + + def test_runner_slots_are_used_as_they_are(self): + for translator, writes_swa in ( + (self.target, True), + (self.private_draft, False), + ): + slots = torch.zeros(2, dtype=torch.int64) + batch = SimpleNamespace( + req_pool_indices=self.rpi, + seq_lens=self.seq_lens, + seq_lens_cpu=self.seq_lens.clone(), + out_cache_loc=slots, + ) + translator.bind_runner_slots(batch) + # Kept by address: a captured graph writes this buffer on replay. + self.assertIs(batch.out_cache_loc, slots) + self.assertIsNone(batch.out_cache_loc_virtual) + self.assertTrue(batch.out_cache_loc_is_physical) + swa = translator.write_ids(batch, _SWA) + if writes_swa: + # The sink in the sliding-window sub-pool too. + self.assertTrue(torch.equal(swa, torch.zeros_like(slots))) + else: + self.assertIsNone(swa) + + def test_a_replay_batch_writes_the_padded_slots(self): + plan = self._plan() + n = plan.write_physical.numel() + slots = torch.zeros(n + 2, dtype=torch.int64) + slots[:n].copy_(plan.write_physical) + batch = SimpleNamespace() + plan.bind_replay(batch, self.target, slots=slots) + self.assertIs(batch.kv_loc_plan, plan) + self.assertIs(batch.out_cache_loc, slots) + self.assertIsNone(batch.out_cache_loc_virtual) + swa = self.target.write_ids(batch, _SWA) + self.assertEqual(swa.shape[0], n + 2) + self.assertTrue(torch.equal(swa[:n], plan.write_ids(self.target, kind=_SWA))) + self.assertEqual(int(swa[n:].abs().sum()), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_physical_write_loc.py b/test/registered/unit/mem_cache/test_physical_write_loc.py index 975b57c545..79b3164790 100644 --- a/test/registered/unit/mem_cache/test_physical_write_loc.py +++ b/test/registered/unit/mem_cache/test_physical_write_loc.py @@ -15,7 +15,7 @@ Under the token-major views a virtual id is in range and, by value, the same kind of integer as a physical one, so a unified pool cannot tell a skipped -rebind from a translated loc by looking at it. `rebind_write_loc` marks the +bind from a translated loc by looking at it. A plan's `bind` marks the batch's loc physical, producers carry the mark in `KVWriteLoc`, composites forward it, and the unified write doors refuse a loc without it. @@ -31,6 +31,7 @@ from types import SimpleNamespace import torch from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind from sglang.srt.mem_cache.memory_pool import ( KVWriteLoc, MHATokenToKVPool, @@ -66,7 +67,11 @@ def _plain_translator(): def _translating_translator(v2p): src = _plain_translator() src.is_translating = True - src._translate_write_full = lambda t, out=None: v2p[t.to(torch.int64)] + src._spaces = { + IdSpaceKind.FULL: IdSpace( + key=(IdSpaceKind.FULL, "test"), write=lambda t: v2p[t.to(torch.int64)] + ) + } return src @@ -111,7 +116,7 @@ class TestRebindMarksTheBatch(unittest.TestCase): def test_non_translating_pool_marks_physical_without_rebinding(self): loc = torch.tensor([3, 5], dtype=torch.int64) fb = _batch(loc) - _plain_translator().rebind_write_loc(fb) + _plain_translator().bind_own_plan(fb) self.assertIs(fb.out_cache_loc, loc) # physical by allocation: untouched self.assertTrue(fb.out_cache_loc_is_physical) @@ -119,13 +124,13 @@ class TestRebindMarksTheBatch(unittest.TestCase): v2p = torch.tensor([7, 6, 5, 4], dtype=torch.int64) loc = torch.tensor([1, 2], dtype=torch.int64) fb = _batch(loc) - _translating_translator(v2p).rebind_write_loc(fb) + _translating_translator(v2p).bind_own_plan(fb) self.assertTrue(torch.equal(fb.out_cache_loc, v2p[loc])) self.assertTrue(fb.out_cache_loc_is_physical) def test_no_loc_stays_unmarked(self): fb = _batch(None) - _translating_translator(torch.arange(4)).rebind_write_loc(fb) + _translating_translator(torch.arange(4)).bind_own_plan(fb) self.assertFalse(fb.out_cache_loc_is_physical) diff --git a/test/registered/unit/mem_cache/test_unified_mha_views.py b/test/registered/unit/mem_cache/test_unified_mha_views.py index 710efe60d4..804246d44a 100644 --- a/test/registered/unit/mem_cache/test_unified_mha_views.py +++ b/test/registered/unit/mem_cache/test_unified_mha_views.py @@ -30,6 +30,7 @@ from types import SimpleNamespace import torch from sglang.srt.environ import envs +from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind from sglang.srt.mem_cache.layout.paged_view import paged_view from sglang.srt.mem_cache.layout.token_major import ENTRY_ALIGN_BYTES, build_dense_views from sglang.srt.mem_cache.memory_pool import KVWriteLoc @@ -550,10 +551,10 @@ class TestFactoryViews(unittest.TestCase): self.assertEqual(pool.k_buffer[0].dtype, torch.uint8) def test_rebind_emits_physical_full_and_build_derives_swa(self): - """End-to-end over the real factory: rebind_write_loc rebinds - out_cache_loc to FULL-side physical ids (phase 1), and the per-batch - build derives the SWA write loc pointwise from those values (phase 2) - -- both checked against the v2p tables over the VIRTUAL ids.""" + """End-to-end over the real factory: a batch's own plan binds + out_cache_loc to FULL-side physical ids and derives the SWA write ids + from the virtual window -- both checked against the v2p tables over + the VIRTUAL ids.""" from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.runtime_context import get_parallel @@ -584,11 +585,11 @@ class TestFactoryViews(unittest.TestCase): device="cpu", ) self.assertTrue(source.is_translating) - source.rebind_write_loc(fb) + source.bind_own_plan(fb) self.assertTrue(torch.equal(fb.out_cache_loc, expected_full)) self.assertTrue( torch.equal( - source.sliding_window_write_loc_for(fb.out_cache_loc), expected_swa + source.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), expected_swa ) ) diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py index 25e878922b..f200784fb0 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py +++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py @@ -21,16 +21,15 @@ page ids: kernel_page(virtual_page) = v2p[virtual_page] -Since the read-path translator, ONE builder computes that formula for every -family — `build_index_table` (the canonical) — and the backends only -differ in how they consume it: - - trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: rows filled straight - into their padded block tables (`KVIndexTranslator.fill_read_table`, prefix-only so - the backends' own -1 / stale tail sentinels survive); - - the flashinfer updaters: token ids reconstructed from the canonical by +ONE builder computes that formula for every family -- the iteration plan's +read table (`KVLocPlan.read_table`) -- and the backends only differ in how +they consume it: + - trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows + copied into their padded block tables (`KVIndexTranslator.copy_page_table`); + - the flashinfer updaters: token ids reconstructed from the table by `create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`; - - fa3's captured decode: `normal_decode_set_metadata` copies the canonical - rows' live prefixes (src_is_read_table=True). + - fa3's captured decode: the plan's rows copied into its captured page + table. Covered here: - the static `create_flashmla_kv_indices_triton` (no id-space knowledge left) @@ -216,6 +215,44 @@ class TestBlockTable(unittest.TestCase): torch.equal(gpu, cpu), f"ps={page_size}:\ngpu={gpu}\ncpu={cpu}" ) + def test_zero_tail_fills_a_fresh_table(self): + """With ``zero_tail`` a fresh (garbage) table comes back whole in one + launch: each row's live prefix, the sink past it up to ``max_pages``, + and nothing past ``max_pages``. The CUDA kernel must agree with the + CPU path.""" + from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table + + for page_size in (1, 32): + rt, rpi, sl, v2p = self._make_batch(page_size) + bs = rpi.shape[0] + max_pages = int((sl.max().item() + page_size - 1) // page_size) + 2 + + def fill(device): + # One guard column past the table catches a spill off the row. + out = torch.full( + (bs, max_pages + 1), 7, dtype=torch.int32, device=device + ) + build_kv_read_table( + req_to_token=rt.to(device), + req_pool_indices=rpi.to(device), + seq_lens=sl.to(device=device, dtype=torch.int64), + v2p=v2p.to(device), + page_size=page_size, + max_pages=max_pages, + out=out, + zero_tail=True, + ) + return out.cpu() + + gpu, cpu = fill(_DEV), fill("cpu") + self.assertTrue(torch.equal(gpu, cpu), f"ps={page_size}") + self.assertTrue(bool((gpu[:, max_pages] == 7).all()), "spilled") + want = _reference(rt, rpi, sl, page_size, v2p=v2p).cpu().to(torch.int32) + for b in range(bs): + live = int((int(sl[b]) + page_size - 1) // page_size) + self.assertTrue(torch.equal(gpu[b, :live], want[b, :live])) + self.assertTrue(bool((gpu[b, live:max_pages] == 0).all())) + def test_static_kernel_matches_reference(self): """The stripped (id-space-free) flashmla kernel is byte-identical to the plain token//ps reference -- guards the v2p-arg removal itself.""" diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index f8e151c95a..5ff28606ac 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -303,10 +303,8 @@ class TestDefaultConfigurator(CustomTestCase): self.assertEqual(mock_calculate_mla_kv_cache_dim.call_count, 2) def test_fused_draft_entry_overrides_the_layer_ratio(self): - """Unified mamba-MHA fusion prices the EXACT fused entry (host + - draft + alignment pad): the per-draft-layer ratio under-reserves the pad, - so a solve over it hands out more tokens than the fused pages hold. - The control pins that path-only drafts keep the ratio arm.""" + """A fused draft prices the exact fused entry (host + draft + pad); + the per-draft-layer ratio under-reserves the pad.""" mr = _make_model_runner(self, speculative_algorithm="EAGLE") mr.spec_algorithm.is_eagle.return_value = True mr.spec_algorithm.is_none.return_value = False @@ -320,12 +318,7 @@ class TestDefaultConfigurator(CustomTestCase): ) fused = create_memory_pool_configurator(mr) - mr.fused_entry_bytes.side_effect = None - mr.fused_entry_bytes.return_value = None - unfused = create_memory_pool_configurator(mr) self.assertEqual(fused._cell_size, 77777) - base = _full_per_token(mr) * 32 - self.assertEqual(unfused._cell_size, int(base * (1 + 1 / 32))) class TestHybridSWAConfigurator(CustomTestCase): @@ -1459,13 +1452,9 @@ class TestSWAPoolFloor(CustomTestCase): class TestFusedDraftPricing(unittest.TestCase): - """Fused draft KV: the boot solve must price the EXACT fused entry. - - The fused full-side entry (host + draft + alignment pad) replaces BOTH the - per-token full term and the per-draft-layer approximation; charging both - would over-reserve, charging only the approximation would under-reserve - and OOM at pool construction (the factory allocates tokens x fused entry). - """ + """A fused full-side entry (host + draft + pad) replaces both the + per-token full term and the per-draft-layer approximation in every solve: + charging both over-reserves, charging only the approximation under-reserves.""" def _make(self, fused_entry, draft_layers, **runner_kwargs): mr = _make_model_runner( @@ -1500,14 +1489,6 @@ class TestFusedDraftPricing(unittest.TestCase): + cfg._draft_cell_size ) - def test_unfused_keeps_the_draft_layer_approximation(self): - _, cfg = self._make(fused_entry=None, draft_layers=2) - self.assertEqual(cfg._draft_full_layers_num, 2) - self.assertEqual( - cfg._cell_size, - self._expected_cell(cfg, cfg._full_per_token * (4 + 2)), - ) - def test_fused_entry_replaces_full_and_draft_terms(self): fused_entry = 54_321 _, cfg = self._make(fused_entry=fused_entry, draft_layers=2) @@ -1515,9 +1496,8 @@ class TestFusedDraftPricing(unittest.TestCase): self.assertEqual(cfg._cell_size, self._expected_cell(cfg, fused_entry)) def _assert_unified_bytes_price_the_fused_entry(self, cfg, config, available): - """The bytes handed to the unified pool must buy every full token at - the fused entry -- the factory divides them by it -- and must not leave - a full token's worth of the budget unspent.""" + """The unified pool's bytes buy every full token at the fused entry + (the factory divides by it) without leaving a full token unspent.""" fused_entry = cfg._fused_full_entry swa_bytes = ( config.swa_max_total_num_tokens * cfg._swa_per_token * cfg._swa_layers_num diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py index ed0906d1b5..c655e375d0 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py @@ -9,6 +9,7 @@ import torch import sglang.srt.model_executor.model_runner_components.cuda_graph_setup as graph_setup import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -31,6 +32,17 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=11, suite="base-a-test-cpu") +def _static_pool_translator(): + """A static pool's translator: its plans hand every batch its own ids.""" + return KVIndexTranslator( + req_to_token=torch.zeros((1, 8), dtype=torch.int32), + token_to_kv_pool_allocator=None, + token_to_kv_pool=object(), + page_size=1, + device="cpu", + ) + + class _FakeAttentionBackend: supports_full_cuda_graph_chunked_prefix = True @@ -168,6 +180,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): prepare_dummy_forward_batch=lambda batch: prepared.append(batch) or batch, attn_tp_sequence_sharded=lambda _: False, attn_backend=attention_backend, + kv_index_translator=_static_pool_translator(), ) runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner.model_runner = model_runner @@ -321,9 +334,11 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): runner.max_context_size = None runner._capture_chunked_prefix = False runner.buffer_registry = _FakeBatchRegistry() + translator = _static_pool_translator() runner.model_runner = SimpleNamespace( attn_tp_sequence_sharded=lambda _: False, prepare_dummy_forward_batch=lambda batch: batch, + kv_index_translator=translator, ) runner.enable_cp_bcg_capture = False runner._is_full_backend = False @@ -357,6 +372,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase): capture_hidden_mode=CaptureHiddenMode.NULL, global_forward_mode=ForwardMode.EXTEND, ) + translator.bind_own_plan(forward_batch) static_batch = runner.load_batch(forward_batch) diff --git a/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py b/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py deleted file mode 100644 index 566cf048cc..0000000000 --- a/test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2023-2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""`resolve_spec_aux_hidden_state_config` runs before the TP group exists. - -BUG REGRESSION. `ModelRunner.init_spec_aux_hidden_state()` runs BEFORE -`init_torch_distributed()`, so the attention-TP group does not exist while -this resolver runs. A `get_parallel().attn_tp_size` read inside it therefore -raises `AssertionError: attention tensor model parallel group is not -initialized` and kills the server at startup -- for EVERY EAGLE-family run -with a draft path, unified pool or not. It is invisible to CPU unit gates -(they never boot a runner) and was caught only by a GPU boot guard. - -The resolver therefore records the draft's `ModelConfig` whole, and the -per-GPU geometry is derived at pool-build time (`draft_kv_profile`), where -the group exists -- the same point the target divides its own heads. - - python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py -v -""" - -import ast -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - -_SRC = Path(_mod.__file__) - - -def _function_named(tree, name): - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef) and node.name == name: - return node - raise AssertionError(f"{name} not found in {_SRC}") - - -def _config(*, num_nextn_predict_layers, num_hidden_layers=1): - return SimpleNamespace( - num_nextn_predict_layers=num_nextn_predict_layers, - num_hidden_layers=num_hidden_layers, - num_attention_layers=num_hidden_layers, - is_hybrid_swa=False, - is_deepseek_v4_arch=False, - ) - - -def _spec_algorithm(*, eagle: bool, eagle3: bool = False): - return SimpleNamespace( - is_eagle=lambda: eagle, - is_standalone=lambda: False, - is_eagle3=lambda: eagle3, - ) - - -class TestResolverTouchesNoParallelState(unittest.TestCase): - def test_no_parallel_state_in_the_eagle_resolver(self): - """AST pin: the EAGLE resolver runs pre-distributed-init, so any - parallel-state read in it -- `get_parallel()` directly, or a per-GPU - head count that divides by it -- is a boot crash, not a wrong number.""" - tree = ast.parse(_SRC.read_text()) - fn = _function_named(tree, "_resolve_eagle_aux_hidden_state") - called = { - n.func.id - for n in ast.walk(fn) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) - } - self.assertNotIn( - "get_parallel", - called, - "_resolve_eagle_aux_hidden_state reads parallel state, but it runs " - "before init_torch_distributed() -- the attention-TP group does not " - "exist yet and the server dies at startup.", - ) - methods = { - n.func.attr - for n in ast.walk(fn) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) - } - self.assertNotIn("get_num_kv_heads", methods) - imported = { - alias.name - for n in ast.walk(fn) - if isinstance(n, ast.ImportFrom) - for alias in n.names - } - self.assertNotIn("get_parallel", imported) - - def _resolve_with(self, *, draft_path, target, draft): - """Run the EAGLE resolver with the config factory stubbed: `draft` - answers the draft-mode read, `target` is the runner's own config.""" - config = _mod.SpecAuxHiddenStateConfig() - spec = SimpleNamespace( - speculative_draft_model_path=draft_path, - speculative_draft_model_revision=None, - ) - with ( - patch.object(_mod, "get_spec", return_value=spec), - patch.object( - _mod.ModelConfig, "from_server_args", return_value=draft - ) as factory, - ): - _mod._resolve_eagle_aux_hidden_state( - config=config, - server_args=None, - model_config=target, - spec_algorithm=_spec_algorithm(eagle=True), - is_draft_worker=False, - ) - self.assertTrue(factory.call_args.kwargs["is_draft_model"]) - return config - - def test_path_less_nextn_fuses_without_resizing_the_private_pool(self): - """Every `num_nextn_predict_layers = 1` assignment in model_config.py - is guarded by `if is_draft_model`, so a path-less MTP target answers - None for it. The fused placement must read the draft-mode config, or a - path-less NEXTN head never fuses; the private-pool budget must keep - reading the target, or every path-less MTP deployment's token budget - changes.""" - target = _config(num_nextn_predict_layers=None) - draft = _config(num_nextn_predict_layers=1) - config = self._resolve_with(draft_path=None, target=target, draft=draft) - self.assertIs(config.draft_model_config, draft) - self.assertEqual(config.draft_kv_num_layers, 1) - self.assertIsNone(config.eagle_draft_num_layers) - - def test_draft_path_sizes_both_from_the_draft_checkpoint(self): - target = _config(num_nextn_predict_layers=None) - draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2) - config = self._resolve_with(draft_path="/draft", target=target, draft=draft) - self.assertIs(config.draft_model_config, draft) - self.assertEqual(config.draft_kv_num_layers, 2) - self.assertEqual(config.eagle_draft_num_layers, 2) - - def test_the_draft_config_is_recorded_whole(self): - """The aux config carries the draft `ModelConfig`, never a pre-divided - geometry: a per-GPU count recorded here would be computed without the - TP group (every unit fixture runs at tp=1, where that is invisible).""" - fields = _mod.SpecAuxHiddenStateConfig.__struct_fields__ - self.assertIn("draft_model_config", fields) - self.assertNotIn("eagle_draft_total_kv_heads", fields) - self.assertNotIn("eagle_draft_kv_head_num", fields) - - -class TestResolverIsInertOffTheTargetEagleArm(unittest.TestCase): - """BUG REGRESSION. The draft-config block is guarded by "EAGLE-family - target"; a rebase left the hybrid-SWA read outside that guard, so every - draft worker and every spec-off runner raised `UnboundLocalError` at - boot, and the EAGLE3 arm's try/except masked the same read into a silent - `None`. Off the target EAGLE arm the resolver must return untouched.""" - - def _resolve(self, *, spec_algorithm, is_draft_worker): - config = _mod.SpecAuxHiddenStateConfig() - _mod._resolve_eagle_aux_hidden_state( - config=config, - server_args=None, - model_config=None, - spec_algorithm=spec_algorithm, - is_draft_worker=is_draft_worker, - ) - return config - - def test_draft_workers_and_spec_off_runners_resolve_nothing(self): - untouched = _mod.SpecAuxHiddenStateConfig() - for spec_algorithm, is_draft_worker in ( - (_spec_algorithm(eagle=True), True), - (_spec_algorithm(eagle=True, eagle3=True), True), - (_spec_algorithm(eagle=False), False), - ): - config = self._resolve( - spec_algorithm=spec_algorithm, is_draft_worker=is_draft_worker - ) - self.assertEqual(config, untouched) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py b/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py new file mode 100644 index 0000000000..84755b272f --- /dev/null +++ b/test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py @@ -0,0 +1,90 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Which config the EAGLE aux-hidden-state resolver sizes the draft KV from. + +python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py -v +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=3, stage="weekly", runner_config="cpu") + + +def _config(*, num_nextn_predict_layers, num_hidden_layers=1): + return SimpleNamespace( + num_nextn_predict_layers=num_nextn_predict_layers, + num_hidden_layers=num_hidden_layers, + num_attention_layers=num_hidden_layers, + is_hybrid_swa=False, + is_deepseek_v4_arch=False, + ) + + +class TestEagleResolverDraftConfig(unittest.TestCase): + def _resolve_with(self, *, draft_path, target, draft): + """Run the EAGLE resolver with the config factory stubbed: `draft` + answers the draft-mode read, `target` is the runner's own config.""" + config = _mod.SpecAuxHiddenStateConfig() + spec = SimpleNamespace( + speculative_draft_model_path=draft_path, + speculative_draft_model_revision=None, + ) + with ( + patch.object(_mod, "get_spec", return_value=spec), + patch.object( + _mod.ModelConfig, "from_server_args", return_value=draft + ) as factory, + ): + _mod._resolve_eagle_aux_hidden_state( + config=config, + server_args=None, + model_config=target, + spec_algorithm=SimpleNamespace( + is_eagle=lambda: True, + is_standalone=lambda: False, + is_eagle3=lambda: False, + ), + is_draft_worker=False, + ) + self.assertTrue(factory.call_args.kwargs["is_draft_model"]) + return config + + def test_path_less_nextn_fuses_without_resizing_the_private_pool(self): + """A path-less MTP head sizes its fused draft KV from the draft-mode + config (the only one that names its NEXTN layers), while the private + draft pool keeps reading the target config, so its token budget is + unchanged.""" + target = _config(num_nextn_predict_layers=None) + draft = _config(num_nextn_predict_layers=1) + config = self._resolve_with(draft_path=None, target=target, draft=draft) + self.assertIs(config.draft_model_config, draft) + self.assertEqual(config.draft_kv_num_layers, 1) + self.assertIsNone(config.eagle_draft_num_layers) + + def test_draft_path_sizes_both_from_the_draft_checkpoint(self): + target = _config(num_nextn_predict_layers=None) + draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2) + config = self._resolve_with(draft_path="/draft", target=target, draft=draft) + self.assertIs(config.draft_model_config, draft) + self.assertEqual(config.draft_kv_num_layers, 2) + self.assertEqual(config.eagle_draft_num_layers, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py b/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py index 96ffd8fe4d..d53d7511cb 100644 --- a/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py +++ b/test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py @@ -13,11 +13,12 @@ # ============================================================================== """ForwardBatch construction wires the unified write-loc rebind. -`init_new` must call `kv_index_translator.rebind_write_loc`: a construction -path that skips it ships VIRTUAL write ids to the kernels, a silent -wrong-slot store. Also runs the REAL `_pad_inputs_to_size` against a live -translator, since pad lanes are zeros and zeros must derive to the slot-0 -sink. Sliding-window semantics are pinned in test_kv_index_translator.py. +`init_new` must bind the batch to a plan (`translator.own_plan`, +`kv_loc_plan.bind`): a construction path that skips it ships VIRTUAL write +ids to the kernels, a silent wrong-slot store. Also runs the REAL +`_pad_inputs_to_size` against a live translator, since pad lanes are zeros +and must write the slot-0 sink in every id space. Sliding-window semantics +are pinned in test_kv_index_translator.py. python -m pytest test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py -v """ @@ -32,6 +33,7 @@ from unittest.mock import create_autospec import torch from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -70,22 +72,25 @@ def _armed_source(v2p, swa_map): ) src.is_translating = True src._translate_full = lambda t, out=None: v2p[t.to(torch.int64)] - # The WRITE loc has its own translate because under DCP it arrives widened; - # at dcp_size == 1 it is the read translate, so arm it with the same fake. - src._translate_write_full = src._translate_full - # Phase 2 derives from physical values through p2v + the swa v2p; arm the - # inverse of the fake v2p (ps=1, so the expected swa loc for virtual t is - # swa_map[t]). - p2v = torch.zeros(int(v2p.max()) + 1, dtype=torch.int64) - p2v[v2p] = torch.arange(v2p.numel(), dtype=torch.int64) - src._full_p2v_table = p2v - src._swa_v2p_table = swa_map + src._spaces = { + # The WRITE loc has its own translate because under DCP it arrives + # widened; at dcp_size == 1 it is the read translate. + IdSpaceKind.FULL: IdSpace( + key=(IdSpaceKind.FULL, "test"), write=src._translate_full + ), + # The sliding-window ids derive from the virtual window (ps=1, so the + # swa id for virtual t is swa_map[t]). + IdSpaceKind.SLIDING_WINDOW: IdSpace( + key=(IdSpaceKind.SLIDING_WINDOW, "test"), + write=lambda t: swa_map[t.to(torch.int64)], + ), + } return src def _call_names(func) -> list: """Dotted call targets appearing in `func`'s body, e.g. - 'model_runner.kv_index_translator.rebind_write_loc'.""" + 'kv_loc_plan.bind'.""" tree = ast.parse(textwrap.dedent(inspect.getsource(func))) names = [] for node in ast.walk(tree): @@ -104,13 +109,15 @@ def _call_names(func) -> list: class TestForwardBatchWiring(CustomTestCase): """Critical-path bookkeeping: the construction-time call sites.""" - def test_init_new_calls_the_rebind(self): - self.assertIn( - "model_runner.kv_index_translator.rebind_write_loc", - _call_names(ForwardBatch.init_new.__func__), - "init_new must rebind the write loc through the source; a batch " - "built without it ships virtual ids to the kernels", - ) + def test_init_new_binds_the_plan(self): + names = _call_names(ForwardBatch.init_new.__func__) + for call in ("translator.own_plan", "kv_loc_plan.bind"): + self.assertIn( + call, + names, + "init_new must take its write ids from a plan; a batch built " + "without one ships virtual ids to the kernels", + ) class TestPadComposesWithDerivation(CustomTestCase): @@ -120,11 +127,11 @@ class TestPadComposesWithDerivation(CustomTestCase): kv_index_translator=src, ) - def test_pad_lanes_derive_to_sink_and_slices_stay_pointwise(self): - """The REAL `_pad_inputs_to_size` composes with phase 2: pad lanes are - zeros, zeros derive to the slot-0 sink, and any slice of the padded - tensor (the TBO-child shape) derives pointwise -- no handover call - exists for the pad to make.""" + def test_pad_lanes_write_the_sink_in_both_spaces(self): + """The REAL `_pad_inputs_to_size` pads the batch's write ids, and the + sliding-window ids its plan hands out follow them: pad lanes write the + slot-0 sink in both spaces, and both stay the same length as the + batch's tokens.""" n, padded = 3, 6 v2p = torch.arange(64, dtype=torch.int64) * 3 swa_map = torch.arange(64, dtype=torch.int64) * 5 @@ -133,36 +140,37 @@ class TestPadComposesWithDerivation(CustomTestCase): fb = _make_fb(virt.clone()) fb.positions = torch.arange(n, dtype=torch.int64) fb.lora_ids = [None] * fb.batch_size - src.rebind_write_loc(fb) + src.bind_own_plan(fb) + swa = IdSpaceKind.SLIDING_WINDOW self.assertTrue(torch.equal(fb.out_cache_loc, v2p[virt])) + self.assertTrue(torch.equal(src.write_ids(fb, swa), swa_map[virt])) fb._pad_inputs_to_size(self._fake_runner_for_pad(src), padded, fb.batch_size) - self.assertEqual(fb.out_cache_loc.shape[0], padded) # Padded tail lanes go to slot 0 -- the reserved dummy-write sink. - self.assertTrue(bool((fb.out_cache_loc[n:] == 0).all())) - loc = src._swa_write_loc_unified(fb.out_cache_loc) - self.assertTrue(torch.equal(loc[:n], swa_map[virt])) - self.assertTrue(bool((loc[n:] == 0).all())) - self.assertEqual(loc.dtype, torch.int64) - # The TBO-child shape: a slice of the PADDED tensor derives pointwise. - sub = src._swa_write_loc_unified(fb.out_cache_loc[1:5]) - self.assertTrue(torch.equal(sub, loc[1:5])) + for loc, want in ( + (fb.out_cache_loc, v2p[virt]), + (src.write_ids(fb, swa), swa_map[virt]), + ): + self.assertEqual(loc.shape[0], padded) + self.assertTrue(torch.equal(loc[:n], want)) + self.assertTrue(bool((loc[n:] == 0).all())) + self.assertEqual(loc.dtype, torch.int64) def test_empty_loc_rebinds_to_empty(self): src = _armed_source( torch.arange(8, dtype=torch.int64), torch.arange(8, dtype=torch.int64) ) fb = _make_fb(torch.empty(0, dtype=torch.int64)) - src.rebind_write_loc(fb) + src.bind_own_plan(fb) self.assertEqual(fb.out_cache_loc.numel(), 0) - self.assertEqual(src._swa_write_loc_unified(fb.out_cache_loc).numel(), 0) + self.assertEqual(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW).numel(), 0) class TestReadRailTranslatesAtProduction(CustomTestCase): - """The model-door READ indices (req_to_token-derived, VIRTUAL under the - unified pool) are translated at their PRODUCTION site -- the cache then - holds the physical result and the pool door never translates.""" + """The model-door READ indices are physical from their PRODUCTION site + (a gather from the iteration's plan under the unified pool) -- the cache + then holds the physical result and the pool door never translates.""" def _fb_for_one_shot(self): fb = _make_fb(torch.tensor([1, 2], dtype=torch.int64)) @@ -172,39 +180,84 @@ class TestReadRailTranslatesAtProduction(CustomTestCase): fb.req_pool_indices = torch.tensor([0, 1], dtype=torch.int64) return fb - def test_one_shot_indices_translated_once_and_cached(self): + def test_one_shot_indices_gathered_from_the_plan_once_and_cached(self): + from unittest.mock import patch + + from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix + + plans = [] + + def pack(plan, *, req_pool_indices, seq_lens, indptr, out, **_): + plans.append(plan) + out.fill_(7) + return True + + fb = self._fb_for_one_shot() + fb.kv_loc_plan = object() + # autospec, not a bare namespace: setting a name the translator does + # not have raises, so renaming the method breaks this test loudly. + fake_translator = create_autospec(KVIndexTranslator, instance=True) + fake_translator.pack_read_stream = pack + fake_backend = SimpleNamespace(kv_index_translator=fake_translator) + with patch.object(mix, "get_attn_backend", return_value=fake_backend): + r1 = fb.fetch_mha_one_shot_kv_indices() + r2 = fb.fetch_mha_one_shot_kv_indices() + + # One gather, from the iteration's plan; the cache holds its result. + self.assertEqual(plans, [fb.kv_loc_plan]) + self.assertIs(r2, r1) + self.assertTrue(torch.equal(r1, torch.full((5,), 7, dtype=torch.int32))) + + def _fb_for_chunks(self): + fb = self._fb_for_one_shot() + fb.kv_loc_plan = object() + fb.num_prefix_chunks = 2 + fb.prefix_chunk_starts = [torch.tensor([0, 0]), torch.tensor([2, 1])] + fb.prefix_chunk_seq_lens = [torch.tensor([2, 1]), torch.tensor([1, 2])] + fb.prefix_chunk_cu_seq_lens = [ + torch.tensor([0, 2, 3], dtype=torch.int32), + torch.tensor([0, 1, 3], dtype=torch.int32), + ] + fb.prefix_chunk_num_tokens = [3, 3] + fb.prefix_chunk_starts_cpu = [[0, 0], [2, 1]] + fb.prefix_chunk_seq_lens_cpu = [[2, 1], [1, 2]] + return fb + + def test_chunk_indices_gathered_from_the_plan(self): + """On a translating pool each prefix chunk's ids come out of the plan's + table: one gather per chunk, at the chunk's start, no translate.""" from unittest.mock import patch from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix calls = [] - sentinel = torch.arange(5, dtype=torch.int64) + 5000 - def translate(t): - calls.append(t) - return sentinel + def pack(plan, *, req_pool_indices, seq_lens, indptr, out, kv_start_idx): + calls.append((plan, seq_lens, indptr, kv_start_idx)) + out.fill_(9) + return True - fb = self._fb_for_one_shot() - fake_pool = SimpleNamespace( - req_to_token=torch.zeros((4, 16), dtype=torch.int32) - ) - # autospec, not a bare namespace: setting a name the translator does - # not have raises, so renaming the method breaks this test loudly. + fb = self._fb_for_chunks() fake_translator = create_autospec(KVIndexTranslator, instance=True) - fake_translator.translate_full_attn_ids = translate + fake_translator.reads_are_translated = True + fake_translator.pack_read_stream = pack fake_backend = SimpleNamespace(kv_index_translator=fake_translator) + fake_pool = SimpleNamespace(req_to_token=torch.zeros((4, 8), dtype=torch.int32)) with ( - patch.object(mix, "get_req_to_token_pool", return_value=fake_pool), patch.object(mix, "get_attn_backend", return_value=fake_backend), - patch.object(mix, "create_flashinfer_kv_indices_triton"), + patch.object(mix, "get_req_to_token_pool", return_value=fake_pool), ): - r1 = fb.fetch_mha_one_shot_kv_indices() - r2 = fb.fetch_mha_one_shot_kv_indices() + fb.prepare_chunked_kv_indices(torch.device("cpu")) - self.assertIs(r1, sentinel) # production site translated - self.assertIs(r2, sentinel) # cache holds the TRANSLATED result - self.assertEqual(len(calls), 1) # translated exactly once - self.assertEqual(calls[0].dtype, torch.int32) # raw producer output + self.assertEqual(len(calls), 2) + for idx, (plan, seq_lens, indptr, kv_start_idx) in enumerate(calls): + self.assertIs(plan, fb.kv_loc_plan) + self.assertIs(seq_lens, fb.prefix_chunk_seq_lens[idx]) + self.assertIs(indptr, fb.prefix_chunk_cu_seq_lens[idx]) + self.assertIs(kv_start_idx, fb.prefix_chunk_starts[idx]) + fake_translator.translate_dcp_read_ids.assert_not_called() + for chunk in fb.prefix_chunk_kv_indices: + self.assertTrue(bool((chunk == 9).all())) def test_one_shot_indices_noop_on_unmigrated_backend(self): from unittest.mock import patch diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py index de06e543a7..bb9db7c723 100644 --- a/test/registered/unit/server_args/test_unified_memory_spec_gate.py +++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py @@ -11,34 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""`--enable-unified-memory` speculative-decoding allow-list. +"""`--enable-unified-memory` speculative-decoding backend allow-lists. -Three audited arms (see `handle_unified_memory_pool`), each with its own -backend constraints: - * DSPARK: the target verifies on the audited verify set (`triton` / - `trtllm_mla` / `cutedsl_mla` / `tokenspeed_mla` / `flashmla` / - `flashinfer` / `fa3`, which includes the MLA family). Its draft fuses - into the target's pages when the fused-draft decision allows, else keeps - a private pool, so the gate leaves the draft backend unconstrained. - * DFLASH: the target verifies on `triton` / `fa3` / `flashinfer`; the MLA - verify family must not leak into this arm. - * EAGLE/EAGLE3: unified targets only (hybrid-SWA or mamba hybrids, either - full-pool kind) -- the draft's KV lives fused inside the full pool's - page envelope (`DenseDraftRegion`), and a draft that does not fuse is - refused at boot. The target's verify set follows the - host kind: the audited verify set on an MLA host, `triton` / - `flashinfer` / `fa3` on an MHA host, and an unresolved backend is - refused. The draft worker (its backend resolves separately: explicit - flag first, else it inherits the target's) runs on `triton` / - `flashinfer` / `fa3`. - -Every arm drafts a linear chain (`--speculative-eagle-topk` in {None, 1}). -Everything else (NGRAM / STANDALONE / registered customs) stays refused. -NGRAM's refusal is load-bearing, not pending: like tree verify it relocates -accepted tokens one at a time inside the target pool, which the unified -pool's page-granular `move_kv_cache` cannot express. Pinned so no arm silently -widens to an unaudited algorithm, family, tree shape, or backend -- and so one -arm's addition never perturbs another. +A speculative backend that does not read through the KV-index translator +reads the unified pool with virtual ids and produces wrong tokens without +crashing, so each arm's admitted backend set is pinned exactly. python -m pytest test/registered/unit/server_args/test_unified_memory_spec_gate.py -v """ @@ -55,14 +32,13 @@ from sglang.srt.configs.model_config import AttentionArch from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu") +register_cpu_ci(est_time=4, suite="base-a-test-cpu") class _PlainHFConfig: - """Matches none of the hybrid-arch isinstance probes.""" + """Matches none of the hybrid-arch probes (`mambaish_config` screens on + the architecture name).""" - # `mambaish_config` screens on the architecture name, so a stub that - # declares none is what "not a hybrid arch" looks like to it. architectures = ["LlamaForCausalLM"] def get_text_config(self): @@ -77,18 +53,10 @@ def _accepts( backend: str | None = "triton", draft_backend: str | None = None, dcp_size: int = 1, - retraction_backup: str | None = None, - fields: dict | None = None, attention_arch: AttentionArch = AttentionArch.MHA, ) -> bool: - """Run just `handle_unified_memory_pool` against a minimal stand-in. - - ServerArgs' real constructor pulls in a model config; this exercises the - single handler under test with the fields it reads (disaggregation off, so - only the speculative / cache / dcp / cuda-graph checks run). ServerArgs is - a msgspec Struct, so a field is restated with `force_setattr`, which the - Struct's own `__setattr__` guard allows and `object.__setattr__` does not. - """ + """Run just `handle_unified_memory_pool` on a ServerArgs whose fields are + restated with `force_setattr` (ServerArgs is a msgspec Struct).""" sa = ServerArgs(model_path="dummy") for name, value in { "enable_unified_memory": True, @@ -98,13 +66,12 @@ def _accepts( "speculative_draft_attention_backend": draft_backend, "enable_hierarchical_cache": False, "enable_lmcache": False, - "disaggregation_decode_retraction_backup": retraction_backup, + "disaggregation_decode_retraction_backup": None, "dcp_size": dcp_size, "cuda_graph_config": None, "attention_backend": backend, "prefill_attention_backend": None, "decode_attention_backend": None, - **(fields or {}), }.items(): msgspec.structs.force_setattr(sa, name, value) sa._model_config = SimpleNamespace( @@ -120,298 +87,103 @@ def _accepts( return False +# The audited verify set (includes the MLA family). +_VERIFY_BACKENDS = ( + "triton", + "trtllm_mla", + "cutedsl_mla", + "tokenspeed_mla", + "flashmla", + "flashinfer", + "fa3", +) +_MHA_RAILS = ("triton", "flashinfer", "fa3") + + class TestUnifiedMemorySpecGate(unittest.TestCase): - # Verify-audited backends for the DSPARK arm (includes the MLA family). - DSPARK_BACKENDS = ( - "triton", - "trtllm_mla", - "cutedsl_mla", - "tokenspeed_mla", - "flashmla", - "flashinfer", - "fa3", - ) - # The translated MHA rails: the EAGLE and DFLASH arms' target backends - # on an MHA host, and every fused draft's backend. - MHA_RAILS = ("triton", "flashinfer", "fa3") - # Algorithms with no audited unified-pool verify rails. - # "NEXTN" is deliberately absent: the CLI alias collapses it to - # "EAGLE" in handle_speculative_decoding BEFORE this gate runs - # (arg_groups/pipeline.py orders the hooks), so the raw string can - # never reach the gate -- a case on it would test an impossible - # input. The aliased spelling is covered by the EAGLE cases. - UNAUDITED_ALGORITHMS = ("NGRAM", "STANDALONE") + VERIFY_BACKENDS = _VERIFY_BACKENDS + MHA_RAILS = _MHA_RAILS + MLA_ONLY = tuple(b for b in _VERIFY_BACKENDS if b not in _MHA_RAILS) + UNAUDITED = ("fa4", "trtllm_mha") - def test_dspark_admitted_on_every_audited_backend(self): - """Each entry is a claim that the backend's spec verify path - translates. Adding one here without the code is how a unified run - silently reads the pool with virtual ids.""" - for backend in self.DSPARK_BACKENDS: - self.assertTrue( - _accepts("DSPARK", backend=backend), - f"DSPARK should pass on verify-audited backend {backend}", - ) - - def test_dspark_refused_on_unaudited_backends(self): - """fa4 and trtllm_mha have no translated spec verify path.""" - for backend in ("fa4", "trtllm_mha"): - self.assertFalse(_accepts("DSPARK", backend=backend)) - - def test_tree_drafting_refused(self): - """Tree verify needs a per-token move inside the TARGET pool, which - the unified pool's page-granular `move_kv_cache` cannot express. - DSPARK demonstrates the rule here; the check itself is unconditional, - so it holds for every admitted arm.""" - for topk in (2, 4, 8): - self.assertFalse(_accepts("DSPARK", topk=topk)) - # The shape knob must not disturb spec-off. - self.assertTrue(_accepts(None, topk=None)) - - def test_dspark_draft_backend_is_not_constrained(self): - """The gate does not constrain DSPARK's draft backend: a draft whose - backend is off the translated rails keeps a private pool (the - fused-draft decision declines fusion), where its translator is a - passthrough and any backend reads correctly. Model hooks declare one - on the operator's behalf -- Kimi-Linear / Kimi-K3 + DSPARK on SM100 - declares `trtllm_mha` -- so refusing it would refuse a configuration - the operator never touched.""" - for draft_backend in (None, *self.DSPARK_BACKENDS, "fa4", "trtllm_mha"): - self.assertTrue( - _accepts("DSPARK", draft_backend=draft_backend), - f"DSPARK draft backend {draft_backend} should pass", - ) + def test_dspark_verify_backends(self): + for backend in self.VERIFY_BACKENDS: + self.assertTrue(_accepts("DSPARK", backend=backend), backend) + for backend in self.UNAUDITED: + self.assertFalse(_accepts("DSPARK", backend=backend), backend) def test_dcp_keeps_flashinfer_off_the_verify_list(self): - """Under --dcp-size > 1 the read ids stay DCP-widened for the consumer - to finish; flashinfer's spec verify gathers its CSR args with no DCP - read translation, so it is refused there while the MLA verify family, - which builds its own DCP block table, still passes.""" - self.assertTrue( - _accepts( - "DSPARK", - backend="trtllm_mla", - dcp_size=2, - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - ) - ) - self.assertFalse( - _accepts( - "DSPARK", - backend="flashinfer", - dcp_size=2, - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - ) - ) - # Without DCP flashinfer stays admitted. - self.assertTrue( - _accepts( - "DSPARK", - backend="flashinfer", - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - ) - ) + """Under DCP flashinfer's spec verify reads DCP-widened ids; the MLA + verify family builds its own DCP block table.""" + mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA) + self.assertTrue(_accepts("DSPARK", backend="trtllm_mla", dcp_size=2, **mla)) + self.assertFalse(_accepts("DSPARK", backend="flashinfer", dcp_size=2, **mla)) + self.assertTrue(_accepts("DSPARK", backend="flashinfer", **mla)) - def test_dflash_admitted_only_on_its_verify_backends(self): - """DFLASH's target verifies on triton / fa3 / flashinfer. fa4 and - trtllm_mha have no translated spec verify path, and the MLA verify - family the DSPARK arm admits must not leak into this arm.""" + def test_dflash_verify_backends(self): + """DFLASH verifies on the MHA rails only; the MLA family DSPARK admits + must not leak into this arm.""" for backend in self.MHA_RAILS: - self.assertTrue( - _accepts("DFLASH", backend=backend), - f"DFLASH should pass on verify-audited backend {backend}", - ) - for backend in ("fa4", "trtllm_mha") + tuple( - b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS - ): - self.assertFalse( - _accepts("DFLASH", backend=backend), - f"DFLASH must be refused on backend {backend}", - ) + self.assertTrue(_accepts("DFLASH", backend=backend), backend) + for backend in self.UNAUDITED + self.MLA_ONLY: + self.assertFalse(_accepts("DFLASH", backend=backend), backend) - def test_eagle_family_admitted_on_hybrid_swa(self): - """EAGLE/EAGLE3 chain on a hybrid-SWA target with audited verify - backends is the fused-draft-KV configuration -- both spellings, - resolved or unset topk, every audited backend.""" + def test_eagle_verify_backends_on_an_mha_host(self): for algorithm in ("EAGLE", "EAGLE3"): for topk in (None, 1): for backend in self.MHA_RAILS: self.assertTrue( _accepts(algorithm, topk=topk, backend=backend), - f"{algorithm} topk={topk} backend={backend} should " - "pass on a hybrid-SWA target", + f"{algorithm} topk={topk} {backend}", ) - - def test_eagle_refused_on_dense_targets(self): - """No fused draft region outside the unified composites: a dense - (non-hybrid) target must be refused, not fail at boot.""" - for algorithm in ("EAGLE", "EAGLE3"): - self.assertFalse(_accepts(algorithm, is_hybrid_swa=False)) - - def test_eagle_admitted_on_mamba_mha(self): - """A mamba hybrid off the MLA backend provisions the fused draft - region in its MHA full sub-pool; refusing it here strands the whole - mamba x EAGLE matrix.""" - with patch.object(kv_cache_hook, "mambaish_config", return_value=object()): - for algorithm in ("EAGLE", "EAGLE3"): - self.assertTrue(_accepts(algorithm, is_hybrid_swa=False)) - - def test_eagle_on_mla_mamba_verifies_on_the_mla_family(self): - """An MLA mamba hybrid fuses into MLA pages, so the target verifies on - the MLA backend family, while the fused draft is MHA-shaped and must run - on the translated MHA rails: an MLA-only backend is refused for the - draft whether named or inherited.""" - with patch.object(kv_cache_hook, "mambaish_config", return_value=object()): - for backend in self.DSPARK_BACKENDS: - self.assertTrue( - _accepts( - "EAGLE", - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - backend=backend, - draft_backend="triton", - ), - f"EAGLE on an MLA host should pass on {backend}", + for backend in self.UNAUDITED + self.MLA_ONLY: + self.assertFalse( + _accepts(algorithm, backend=backend, draft_backend="triton"), + f"{algorithm} {backend}", ) + + def test_eagle_draft_backend(self): + """An unset draft backend inherits the target's; an explicit one must + be on the MHA rails.""" + self.assertTrue(_accepts("EAGLE", draft_backend=None)) + for draft_backend in self.MHA_RAILS: + self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend)) + for draft_backend in self.UNAUDITED: + self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend)) + + def test_eagle_on_an_mla_host(self): + """An MLA mamba hybrid verifies on the audited verify set, while its + fused draft is MHA-shaped: an MLA-only draft backend is refused whether + named or inherited.""" + mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA) + with patch.object(kv_cache_hook, "mambaish_config", return_value=object()): + for backend in self.VERIFY_BACKENDS: + self.assertTrue( + _accepts("EAGLE", backend=backend, draft_backend="triton", **mla), + backend, + ) + self.assertFalse( + _accepts("EAGLE", backend="fa4", draft_backend="triton", **mla) + ) for backend in ("trtllm_mla", "flashmla"): for draft_backend in (None, backend): self.assertFalse( _accepts( - "EAGLE", - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - backend=backend, - draft_backend=draft_backend, + "EAGLE", backend=backend, draft_backend=draft_backend, **mla ), - f"MLA-only draft backend {draft_backend or backend}", + draft_backend or backend, ) - for backend in (None, "fa4"): - self.assertFalse( - _accepts( - "EAGLE", - is_hybrid_swa=False, - attention_arch=AttentionArch.MLA, - backend=backend, - ) - ) - def test_eagle_refused_tree_topk(self): - """Tree verify is not audited for the unified pool; only a linear - chain passes.""" - for topk in (2, 4, 8): - self.assertFalse(_accepts("EAGLE", topk=topk)) - - def test_eagle_refused_unaudited_backends(self): - """Unaudited backends stay out, and the MLA verify set from the - DSPARK arm must not leak into the EAGLE arm.""" - for backend in ("fa4", "trtllm_mha") + tuple( - b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS - ): - self.assertFalse(_accepts("EAGLE", backend=backend)) - - def test_eagle_refused_unset_backend(self): - """A real boot resolves the default backend before this gate runs, so - an unset one here is unresolved: the gate refuses it rather than - trusting it.""" - self.assertFalse(_accepts("EAGLE", backend=None)) - - def test_eagle_draft_backend_pinned(self): - """The draft worker resolves its own backend: unset inherits the - target's (triton here), an explicit triton / flashinfer / fa3 passes, - and anything else refuses.""" - self.assertTrue(_accepts("EAGLE", draft_backend=None)) - for draft_backend in self.MHA_RAILS: - self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend)) - for draft_backend in ("fa4", "trtllm_mha"): - self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend)) - - def test_hierarchical_cache_refused_with_an_eagle_draft(self): - """Under HiCache an EAGLE draft keeps a private pool, and an MTP - head's KV would be packed into the target's page-envelope host pool, - which refuses per-layer draft loads. Without a draft HiCache is not - this gate's to refuse, and with DSPARK HiCache declines draft fusion, - so the draft keeps its private pool.""" - hicache = {"enable_hierarchical_cache": True} - for algorithm in ("EAGLE", "EAGLE3"): - self.assertFalse(_accepts(algorithm, fields=hicache)) - self.assertTrue(_accepts(None, fields=hicache)) - self.assertTrue(_accepts("DSPARK", fields=hicache)) - - def test_pd_decode_refusals_raise_their_own_assertions(self): - """The PD-decode branches screen the model with `mambaish_config` and - refuse with their own assertion, not an error from the screen.""" - pd_decode = { - "disaggregation_mode": "decode", - "disaggregation_transfer_backend": "mooncake", - "pp_size": 1, - "enable_hisparse": False, - } - for extra in ( - { - "disaggregation_decode_enable_offload_kvcache": True, - "hicache_storage_backend": "file", - }, - {"disaggregation_decode_retraction_backup": "host_pool"}, - ): - with patch.object(kv_cache_hook, "mambaish_config", return_value=object()): - self.assertFalse( - _accepts(None, is_hybrid_swa=False, fields={**pd_decode, **extra}) - ) - - def test_host_pool_retraction_refused_with_an_eagle_draft(self): - """The host-pool retraction backup builds the draft's host pool off - its device pool, which an EAGLE draft, always fused on the unified - pool, does not have. Without a draft the backup is not this gate's to - refuse, and with DFLASH or DSPARK it declines draft fusion instead.""" - for algorithm in ("EAGLE", "EAGLE3"): - with self.subTest(algorithm=algorithm): - self.assertTrue(_accepts(algorithm)) - self.assertFalse(_accepts(algorithm, retraction_backup="host_pool")) - for algorithm in (None, "DFLASH", "DSPARK"): - with self.subTest(algorithm=algorithm): - self.assertTrue(_accepts(algorithm, retraction_backup="host_pool")) - - def test_dspark_arm_unchanged(self): - """The EAGLE addition must not perturb DSPARK: its MLA verify set - passes, its chain constraint holds, and unaudited backends refuse.""" - for backend in self.DSPARK_BACKENDS: - self.assertTrue( - _accepts("DSPARK", backend=backend), - f"DSPARK should pass on verify-audited backend {backend}", - ) - self.assertFalse(_accepts("DSPARK", backend="fa4")) - self.assertFalse(_accepts("DSPARK", topk=4)) - - def test_ngram_refused_because_it_relocates_target_kv_per_token(self): - """BUG REGRESSION. NGRAM was admitted on the reasoning that - it carries no draft KV, so only the target-verify rails matter. But a - tree-shaped drafter finalizes a batch through - `move_accept_tokens_to_target_kvcache`, which relocates INDIVIDUAL - accepted tokens inside the target pool -- and the unified pool's - `move_kv_cache` is compaction-only (whole page envelopes). Every - unified NGRAM cell died there: a reshape past the end on the MHA/mamba - hosts, and the SWA composite's explicit refusal on gpt-oss. Admitting - it again without a per-token move primitive re-breaks every family.""" - for backend in self.DSPARK_BACKENDS: - self.assertFalse( - _accepts("NGRAM", backend=backend), - f"NGRAM must stay refused (backend={backend})", - ) - - def test_spec_off_admitted(self): - """The gate constrains only speculative configurations; spec-off must - keep booting regardless of family.""" - for is_hybrid_swa in (True, False): - self.assertTrue(_accepts(None, is_hybrid_swa=is_hybrid_swa)) - - def test_unaudited_algorithms_refused(self): - """Every other algorithm stays out until its verify id rails are - audited -- on every family.""" - for algorithm in self.UNAUDITED_ALGORITHMS: + def test_unaudited_algorithms_and_tree_drafting_refused(self): + """NGRAM and tree drafting relocate accepted tokens one at a time + inside the target pool, which the unified pool's page-granular move + cannot express.""" + for algorithm in ("NGRAM", "STANDALONE"): for is_hybrid_swa in (True, False): self.assertFalse(_accepts(algorithm, is_hybrid_swa=is_hybrid_swa)) + for algorithm in ("DSPARK", "EAGLE"): + for topk in (2, 4): + self.assertFalse(_accepts(algorithm, topk=topk), f"{algorithm} {topk}") if __name__ == "__main__": diff --git a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py index 4e5051f9e1..6eab95252f 100644 --- a/test/registered/unit/spec/test_dp_spec_prefill_coordination.py +++ b/test/registered/unit/spec/test_dp_spec_prefill_coordination.py @@ -281,6 +281,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase): hidden_states=object(), mm_input_embeds=None ), next_token_ids=object(), + kv_loc_plan=None, ) verify_input = object() @@ -306,7 +307,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase): self.assertEqual(kwargs["grammar_barrier"], "grammar") return result - def extend(current, *args): + def extend(current, *args, **kwargs): record("draft_extend", current) return object() diff --git a/test/registered/unit/spec/test_dspark_target_hidden_projection.py b/test/registered/unit/spec/test_dspark_target_hidden_projection.py index 5ea002c44d..b403b62f95 100644 --- a/test/registered/unit/spec/test_dspark_target_hidden_projection.py +++ b/test/registered/unit/spec/test_dspark_target_hidden_projection.py @@ -93,9 +93,7 @@ class DSparkTargetHiddenProjectionTest(CustomTestCase): draft_model=draft_model, draft_model_runner=SimpleNamespace( token_to_kv_pool=pool, - kv_index_translator=SimpleNamespace( - translate_full_attn_ids=lambda ids: ids - ), + kv_index_translator=SimpleNamespace(), ), model_runner=SimpleNamespace(device=torch.device("cpu")), device=torch.device("cpu"), diff --git a/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py b/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py index ba35f2612f..bcfc5f8435 100644 --- a/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py +++ b/test/registered/unit/spec/test_eagle_worker_v2_topk1_fastpath.py @@ -229,7 +229,9 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase): max_context_len=1, ) worker.target_worker = SimpleNamespace( - model_runner=SimpleNamespace(attn_backend=attn_backend) + model_runner=SimpleNamespace( + attn_backend=attn_backend, kv_index_translator=None + ) ) draft_input = SimpleNamespace( bonus_tokens=torch.zeros((1,), dtype=torch.long, device=DEVICE), @@ -253,7 +255,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase): ), patch( "sglang.srt.speculative.eagle_worker_v2.prepare_for_draft", - return_value=(forward_batch, True), + return_value=(forward_batch, True, None), ), ): worker.draft(batch) diff --git a/test/registered/unit/spec/test_hand_built_view_write_rail.py b/test/registered/unit/spec/test_hand_built_view_write_rail.py deleted file mode 100644 index 2a2558ca42..0000000000 --- a/test/registered/unit/spec/test_hand_built_view_write_rail.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2023-2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A hand-built ForwardBatch view must carry the write rail. - -BUG REGRESSION. The speculative draft-extend cuda-graph runners do not hand a -real ForwardBatch to `init_forward_metadata_out_graph`; they build a -`SimpleNamespace` view over their capture-stable buffers. On a translating -pool that hook reaches `fill_capture_write_loc`, which reads -`out_cache_loc_virtual` -- the pre-translate write loc -- and translates it -into the backend's capture buffer. - -A view that omits the field fails two ways, and the quiet one is worse. A -`SimpleNamespace` raises AttributeError, which at least crashes the cell. But -`fill_capture_write_loc` also accepts the field being None, and answers it by -zeroing the buffer -- page-0 sink ids. So a view that "fixes" the crash by -passing None sends every draft-extend KV write to the sink, and speculative -decoding rejects the resulting drafts instead of failing: it surfaces as -accept length decaying toward 1.0, not as an error. - -Three runners had the omission independently, so this guards the shape rather -than the three sites: any `SimpleNamespace` view that reaches -`init_forward_metadata_out_graph` must name `out_cache_loc_virtual`. - -The field is only read on a unified (translating) pool, which is why the -default builds never caught it. - -CPU-only, no torch. - - python -m pytest test/registered/unit/spec/test_hand_built_view_write_rail.py -v -""" - -import ast -import pathlib -import unittest - -from sglang.test.ci.ci_register import register_cpu_ci - -register_cpu_ci(est_time=2, suite="base-a-test-cpu") - -_SPEC_DIR = ( - pathlib.Path(__file__).resolve().parents[4] - / "python" - / "sglang" - / "srt" - / "speculative" -) -_HOOK = "init_forward_metadata_out_graph" -_RAIL = "out_cache_loc_virtual" - - -def _views_missing_the_rail(path: pathlib.Path): - """(function, variable, lineno) per hand-built view that drops the rail. - - A view counts when it is a `SimpleNamespace(...)` bound to a name that is - later passed to the hook in the same function -- that is exactly the shape - the runners use, and it keeps the walk from flagging namespaces built for - unrelated purposes. - """ - tree = ast.parse(path.read_text()) - offenders = [] - for fn in ast.walk(tree): - if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): - continue - # Names handed to the hook anywhere in this function. - handed = set() - for node in ast.walk(fn): - if ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == _HOOK - ): - for arg in node.args: - if isinstance(arg, ast.Name): - handed.add(arg.id) - if not handed: - continue - for node in ast.walk(fn): - if not (isinstance(node, ast.Assign) and isinstance(node.value, ast.Call)): - continue - func = node.value.func - name = ( - func.attr - if isinstance(func, ast.Attribute) - else getattr(func, "id", None) - ) - if name != "SimpleNamespace": - continue - for target in node.targets: - if not (isinstance(target, ast.Name) and target.id in handed): - continue - # A literal None is the quiet failure: the hook zeroes the - # capture buffer, routing every write to the page-0 sink. - if not any( - kw.arg == _RAIL - and not ( - isinstance(kw.value, ast.Constant) and kw.value.value is None - ) - for kw in node.value.keywords - ): - offenders.append((fn.name, target.id, node.lineno)) - return sorted(offenders, key=lambda o: o[2]) - - -class TestHandBuiltViewWriteRail(unittest.TestCase): - def test_every_hand_built_view_carries_the_rail(self): - offenders = [] - for path in sorted(_SPEC_DIR.rglob("*.py")): - for fn, var, lineno in _views_missing_the_rail(path): - offenders.append(f"{path.name}:{lineno} {fn}() -> {var}") - self.assertEqual( - offenders, - [], - f"hand-built ForwardBatch view reaches {_HOOK} without {_RAIL}; on a " - "unified pool the draft-extend writes land on the page-0 sink: " - + "; ".join(offenders), - ) - - def test_detector_catches_a_view_that_drops_the_rail(self): - """The detector must catch the shape it guards, else it passes green - forever after a refactor of the walk above.""" - src = ( - "class R:\n" - " def replay(self, fb):\n" - " v = SimpleNamespace(\n" - " batch_size=8, out_cache_loc=self.buffers.out_cache_loc\n" - " )\n" - " self.backend.init_forward_metadata_out_graph(v)\n" - ) - self.assertEqual( - _views_missing_the_rail(self._probe(src)), [("replay", "v", 3)] - ) - - def test_detector_catches_a_view_that_passes_none(self): - """None is accepted by `fill_capture_write_loc` and becomes the page-0 - sink, so naming the rail with None is the omission in disguise.""" - src = ( - "class R:\n" - " def replay(self, fb):\n" - " v = SimpleNamespace(batch_size=8, out_cache_loc_virtual=None)\n" - " self.backend.init_forward_metadata_out_graph(v)\n" - ) - self.assertEqual( - _views_missing_the_rail(self._probe(src)), [("replay", "v", 3)] - ) - - def test_detector_accepts_a_view_that_carries_it(self): - """...and must NOT fire once the rail is named, else it is unfixable.""" - src = ( - "class R:\n" - " def replay(self, fb):\n" - " v = SimpleNamespace(\n" - " batch_size=8,\n" - " out_cache_loc=self.buffers.out_cache_loc,\n" - " out_cache_loc_virtual=fb.out_cache_loc_virtual,\n" - " )\n" - " self.backend.init_forward_metadata_out_graph(v)\n" - ) - self.assertEqual(_views_missing_the_rail(self._probe(src)), []) - - def test_detector_ignores_a_namespace_that_never_reaches_the_hook(self): - """A SimpleNamespace built for something else is not a view.""" - src = ( - "class R:\n" - " def replay(self, fb):\n" - " cfg = SimpleNamespace(batch_size=8)\n" - " return cfg\n" - ) - self.assertEqual(_views_missing_the_rail(self._probe(src)), []) - - def _probe(self, src): - # A temp dir, not the CWD: the checkout may be read-only. - import shutil - import tempfile - - scratch = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, scratch, ignore_errors=True) - tmp = pathlib.Path(scratch) / (self.id().replace(".", "_") + ".py") - tmp.write_text(src) - return tmp - - -if __name__ == "__main__": - unittest.main()