From 08d30f230066afec39295f49edf9b317879a9d70 Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Thu, 8 Oct 2026 00:52:21 -0700 Subject: [PATCH] [unified-memory] Speculative decoding on the unified memory pool: translate every read and write path (#37627) Co-authored-by: Caihua Li Co-authored-by: Cheng Wan --- .../kernels/ops/kvcache/kv_read_table.py | 19 +- python/sglang/srt/arg_groups/kv_cache_hook.py | 64 ++++-- .../srt/layers/attention/aiter_backend.py | 27 ++- .../attention/flashattention_backend.py | 129 ++++++++--- .../layers/attention/flashinfer_backend.py | 22 +- .../attention/flashinfer_mla_backend.py | 20 +- .../srt/layers/attention/flashmla_backend.py | 64 ++++-- .../srt/mem_cache/kv_index_translator.py | 41 ++++ python/sglang/srt/models/falcon_h1.py | 1 + python/sglang/srt/speculative/dflash_info.py | 27 ++- python/sglang/srt/speculative/eagle_info.py | 54 ++--- python/sglang/srt/speculative/ngram_info.py | 33 +-- .../test_flashattention_graph_metadata.py | 60 +++++ .../test_verify_widening_is_wired.py | 208 ++++++++++++++++++ .../mem_cache/test_kv_index_translator.py | 59 +++++ .../mem_cache/test_unified_mla_block_table.py | 71 ++++++ .../test_unified_memory_spec_gate.py | 163 ++++++++++++++ 17 files changed, 895 insertions(+), 167 deletions(-) create mode 100644 test/registered/unit/layers/attention/test_verify_widening_is_wired.py create mode 100644 test/registered/unit/server_args/test_unified_memory_spec_gate.py diff --git a/python/sglang/kernels/ops/kvcache/kv_read_table.py b/python/sglang/kernels/ops/kvcache/kv_read_table.py index d3535ad38d..c00201c32e 100644 --- a/python/sglang/kernels/ops/kvcache/kv_read_table.py +++ b/python/sglang/kernels/ops/kvcache/kv_read_table.py @@ -71,6 +71,8 @@ def build_kv_read_indices_kernel( req_stride, # runtime: req_to_token row stride (elements) out_stride, # runtime: uniform row stride, used when row_starts is null item_stride, # runtime: items one program advances per loop trip + seq_len_delta, # runtime: verify widening added to every row's live prefix + max_row_items, # runtime: uniform row width; a longer window stops there PAGE_SIZE: tl.constexpr, EMIT_PER_TOKEN: tl.constexpr, OUT_INT64: tl.constexpr, @@ -78,7 +80,7 @@ def build_kv_read_indices_kernel( ): bid = tl.program_id(0) req = tl.load(req_pool_indices_ptr + bid).to(tl.int64) - seqlen = tl.load(seq_lens_ptr + bid) + seqlen = tl.load(seq_lens_ptr + bid) + seq_len_delta # Derived here, not on the host: one elementwise op there costs a whole # launch, which a captured graph then replays every step. if EMIT_PER_TOKEN: @@ -93,6 +95,9 @@ def build_kv_read_indices_kernel( row_out = out_ptr + tl.load(row_starts_ptr + bid).to(tl.int64) else: row_out = out_ptr + bid.to(tl.int64) * out_stride + # A widened window can outrun a fixed-width row (a verify near the + # context limit); writing past it would land in the next row. + n_items = tl.minimum(n_items, max_row_items) for start in range(tl.program_id(1) * BLOCK, n_items, item_stride): item = start + tl.arange(0, BLOCK) @@ -132,6 +137,7 @@ def _launch( row_starts: Optional[torch.Tensor], kv_start_idx: Optional[torch.Tensor], emit_per_token: bool, + seq_len_delta: int = 0, ) -> None: bs = int(req_pool_indices.numel()) item_programs = min( @@ -148,6 +154,8 @@ def _launch( req_to_token.stride(0), out_stride, item_programs * _BLOCK_ITEMS, + seq_len_delta, + max_items, PAGE_SIZE=page_size, EMIT_PER_TOKEN=emit_per_token, OUT_INT64=out.dtype == torch.int64, @@ -179,12 +187,18 @@ def build_kv_read_table( page_size: int, max_pages: int, out: torch.Tensor, + seq_len_delta: int = 0, ) -> torch.Tensor: """Fill ``out``'s live prefix with PAGE TABLE entries. ``out`` is caller-owned (fresh zeros for the eager path, the module's capture-stable buffer for replay) and only its ``[:bs, :max_pages]`` region's live prefix is written -- never rebound, never tail-cleared. + + ``seq_len_delta`` widens every row's live prefix -- the whole-sequence + verify contract (draft KV read back from the pool). Every row stops at + ``max_pages``: a widened window that outruns the row is cut there rather + than spilling into the next row, on CUDA as on CPU. """ bs = int(req_pool_indices.numel()) assert out.dtype == torch.int32, ( @@ -205,7 +219,7 @@ def build_kv_read_table( if not req_to_token.is_cuda: cols = torch.arange(max_pages, device=req_to_token.device) for b in range(bs): - n_pages = (int(seq_lens[b]) + page_size - 1) // page_size + n_pages = (int(seq_lens[b]) + seq_len_delta + page_size - 1) // page_size live = min(n_pages, max_pages) out[b, :live] = _entries( req_to_token=req_to_token, @@ -228,6 +242,7 @@ def build_kv_read_table( row_starts=None, kv_start_idx=None, emit_per_token=False, + seq_len_delta=seq_len_delta, ) return out diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 85b772b3fe..64ee40ff4c 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -474,6 +474,44 @@ def handle_cache_compatibility(server_args: Any) -> None: ) +# Backends whose speculative verify id rails are translation-audited for +# the unified pool. +_SPEC_VERIFY_AUDITED_BACKENDS = frozenset( + { + "triton", + "trtllm_mla", + "cutedsl_mla", + "tokenspeed_mla", + "flashmla", + "flashinfer", + "fa3", + } +) + + +def _assert_spec_verify_backends(server_args: Any, *, algorithm: str) -> None: + """Refuse target backends whose verify id rails are not translation-audited. + + Only the target's prefill/decode pair is checked. A draft with a KV pool + of its own indexes that pool directly by virtual id, so its translator is + a passthrough and its backend has nothing to translate.""" + allowed = _SPEC_VERIFY_AUDITED_BACKENDS + if resolving_view(server_args).dcp_size > 1: + # flashinfer's spec verify gathers its CSR args with no DCP read + # translation (`translate_dcp_read_ids`); the MLA verify family builds + # its DCP block table itself. + allowed = allowed - {"flashinfer"} + backends = set(attention_backends_of(resolved_view(server_args))) + backends.discard(None) + assert backends <= allowed, ( + f"--enable-unified-memory + {algorithm} requires spec-verify-audited " + f"attention backends {sorted(allowed)} for both prefill " + f"and decode; got {sorted(backends)}. Other backends do " + "not translate speculative verify indices to the unified " + "pool's physical ids yet." + ) + + def handle_unified_memory_pool(server_args: Any) -> None: cfg = resolving_view(server_args) @@ -540,25 +578,15 @@ def handle_unified_memory_pool(server_args: Any) -> None: "audited for the unified pool's virtual-to-physical loc translation. Got " f"--speculative-algorithm={cfg.speculative_algorithm!r}." ) + assert cfg.speculative_eagle_topk in (None, 1), ( + "--enable-unified-memory supports a linear draft chain only " + "(--speculative-eagle-topk in {None, 1}); tree verify relocates " + "accepted tokens one at a time inside the target pool, which the " + "unified pool's page-granular move_kv_cache cannot express. Got " + f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}." + ) if cfg.speculative_algorithm == "DSPARK": - assert cfg.speculative_eagle_topk in (None, 1), ( - "--enable-unified-memory + DSPARK supports a linear draft " - "chain only (--speculative-eagle-topk in {None, 1}); tree " - "verify is not audited for the unified pool. Got " - f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}." - ) - # Both roles: verify routes to either backend depending on - # --speculative-attention-mode. - spec_allowed = {"triton", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"} - spec_backends = set(attention_backends_of(resolved_view(server_args))) - spec_backends.discard(None) - assert spec_backends <= spec_allowed, ( - "--enable-unified-memory + DSPARK requires spec-verify-audited " - f"attention backends {sorted(spec_allowed)} for both prefill " - f"and decode; got {sorted(spec_backends)}. flashinfer / fa3 do " - "not translate speculative verify indices to the unified " - "pool's physical ids yet." - ) + _assert_spec_verify_backends(server_args, algorithm="DSPARK") assert not cfg.enable_two_batch_overlap, ( "--enable-unified-memory does not support --enable-two-batch-overlap: " "TBO's replay split hands each child a view without the pre-translate " diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 51b0131371..f3537946a0 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -344,6 +344,7 @@ class AiterAttnBackend(AttentionBackend): self.kv_cache_dtype = model_runner.kv_cache_dtype self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.kv_index_translator = model_runner.kv_index_translator self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA @@ -1946,10 +1947,10 @@ class AiterAttnBackend(AttentionBackend): else: kv_indices, kv_indptr, qo_indptr, _ = ( forward_batch.spec_info.generate_attn_arg_prefill( - forward_batch.req_pool_indices, - forward_batch.seq_lens, - forward_batch.seq_lens_sum, - self.req_to_token, + req_pool_indices=forward_batch.req_pool_indices, + paged_kernel_lens=forward_batch.seq_lens, + paged_kernel_lens_sum=forward_batch.seq_lens_sum, + translator=self.kv_index_translator, ) ) self.forward_metadata = ForwardMetadata( @@ -4438,6 +4439,7 @@ class AiterIndicesUpdaterPrefill: self.kv_last_page_len = attn_backend.kv_last_page_len self.qo_indptr = attn_backend.qo_indptr self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.kv_index_translator = model_runner.kv_index_translator self.update = self.update_single_wrapper self.kv_indices = None @@ -4509,10 +4511,10 @@ class AiterIndicesUpdaterPrefill: else: kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - paged_kernel_lens, - paged_kernel_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=paged_kernel_lens, + paged_kernel_lens_sum=paged_kernel_lens_sum, + translator=self.kv_index_translator, ) ) @@ -4526,6 +4528,7 @@ class AiterMlaIndicesUpdaterPrefill: # Buffers and wrappers self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.kv_index_translator = model_runner.kv_index_translator self.update = self.update_single_wrapper self.kv_indptr = None @@ -4587,10 +4590,10 @@ class AiterMlaIndicesUpdaterPrefill: else: kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - kv_lens, - kv_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=kv_lens, + paged_kernel_lens_sum=kv_lens_sum, + translator=self.kv_index_translator, ) ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 0e276e9b3c..d0c2f40390 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -465,6 +465,12 @@ class FlashAttentionBackend(AttentionBackend): # the persistent metadata. if not forward_batch.forward_mode.is_draft_extend_v2(): return + if self.kv_index_translator.is_translating: + # The draft-extend runners record this call unconditionally, but a + # translating backend rebuilds these tables out of graph on every + # replay; a captured raw req_to_token gather would then overwrite + # them with virtual ids. + return bs = forward_batch.batch_size metadata = self.draft_extend_metadata[bs] mapping = self._in_graph_full_to_swa_index_mapping() @@ -511,6 +517,10 @@ class FlashAttentionBackend(AttentionBackend): return self.token_to_kv_pool.full_to_swa_index_mapping def draft_extend_metadata_captured_in_graph(self) -> bool: + # A translating backend rebuilds out of graph: the captured gather + # would bake raw req_to_token (virtual) ids into the page table. + if self.kv_index_translator.is_translating: + return False return ( not self.use_sliding_window_kv_pool or self.token_to_kv_pool.full_to_swa_index_mapping is not None @@ -813,6 +823,32 @@ class FlashAttentionBackend(AttentionBackend): m.max_seq_len_k = self.max_context_len self.forward_metadata = m + def _spec_read_seq_len_delta( + self, forward_mode: ForwardMode, spec_info: Optional[SpecInput] + ) -> int: + """Columns past ``seq_lens`` this mode's whole-sequence read covers — + the translated table's fill must reach ``cache_seqlens``. Takes the + two fields rather than a ForwardBatch: the captured build slices its + batch and has none. Now read by both builds, so eager and replay + widen identically. + + Zero for the prefix-only shapes: normal decode/extend, the topk>1 + split (drafts read via the expand metadata), draft-extend whose lens + already include the window, and draft-extend's idle batch. The ragged + verify layout's per-row lens are bounded by the draft window, so its + upper bound rides the same delta.""" + if spec_info is None: + return 0 + if forward_mode.is_target_verify() and self.topk <= 1: + return self.speculative_num_draft_tokens + if ( + forward_mode.is_decode_or_idle() + and self.topk <= 1 + and self.speculative_num_steps > 0 + ): + return self.speculative_step_id + 1 + return 0 + def init_forward_metadata(self, forward_batch: ForwardBatch): """Initialize forward metadata hence all layers in the forward pass can reuse it.""" metadata = FlashAttentionMetadata() @@ -1201,7 +1237,15 @@ class FlashAttentionBackend(AttentionBackend): else None ) if _unified_read: - kv_view = self.kv_index_translator.index_table_for_batch(forward_batch) + delta = self._spec_read_seq_len_delta( + forward_batch.forward_mode, forward_batch.spec_info + ) + if delta: + kv_view = self.kv_index_translator.widened_index_table( + forward_batch, seq_len_delta=delta + ) + else: + kv_view = self.kv_index_translator.index_table_for_batch(forward_batch) metadata.page_table = kv_view.ids if self.use_sliding_window_kv_pool: metadata.swa_page_table = kv_view.sliding_window_ids @@ -2892,13 +2936,11 @@ class FlashAttentionBackend(AttentionBackend): # Page table built on-device (self-guards on cache_seqlens); # max_seq_len_k left unset -- unread here (scheduler_metadata # is normal-decode-only). - # Spec is asserted off under the unified pool, so this - # captured view is always the passthrough (req_to_token). self._set_decode_page_metadata( metadata, req_pool_indices, seq_lens, - self.speculative_step_id + 1, + self._spec_read_seq_len_delta(forward_mode, spec_info), ) else: @@ -2999,7 +3041,10 @@ class FlashAttentionBackend(AttentionBackend): else self.max_context_len ) self._set_decode_page_metadata( - metadata, req_pool_indices, seq_lens, 0 + metadata, + req_pool_indices, + seq_lens, + self._spec_read_seq_len_delta(forward_mode, spec_info), ) if self.local_attn_builder is not None: @@ -3034,32 +3079,20 @@ class FlashAttentionBackend(AttentionBackend): geometry = build_ragged_target_verify_geometry( seq_lens=seq_lens, layout=padded ) - metadata.cache_seqlens_int32.copy_(geometry.cache_seqlens_int32) metadata.cu_seqlens_q.copy_(geometry.cu_seqlens_q) + # Per-row verify lens; the builder re-derives + # cache_seqlens from them at delta 0. + verify_lens = geometry.cache_seqlens_int32 + verify_delta = 0 else: - metadata.cache_seqlens_int32.copy_( - (seq_lens + self.speculative_num_draft_tokens) - ) + verify_lens = seq_lens + verify_delta = self.speculative_num_draft_tokens # Page table built on-device (self-guards on cache_seqlens); # max_seq_len_k left unset -- unread here (scheduler_metadata is # normal-decode-only). - metadata.cu_seqlens_k[1:].copy_( - torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) - ) - has_swa = self.use_sliding_window_kv_pool - build_trtllm_mha_page_table( - req_to_token=self.req_to_token, - req_pool_indices=req_pool_indices, - cache_seqlens=metadata.cache_seqlens_int32, - page_table=metadata.page_table, - page_size=self.page_size, - swa_page_table=metadata.swa_page_table if has_swa else None, - full_to_swa=( - self.token_to_kv_pool.full_to_swa_index_mapping - if has_swa - else None - ), + self._set_decode_page_metadata( + metadata, req_pool_indices, verify_lens, verify_delta ) else: # When topk > 1, we need two specific target verify metadata, and then merge states @@ -3191,18 +3224,44 @@ class FlashAttentionBackend(AttentionBackend): max_seq_pages = ( metadata.max_seq_len_k + self.page_size - 1 ) // self.page_size - page_indices = self.req_to_token[ - req_pool_indices[:, None], - self.draft_extend_metadata["strided_indices"][:max_seq_pages], - ] - if self.use_sliding_window_kv_pool and metadata.swa_page_table is not None: - swa_page_indices = self.token_to_kv_pool.translate_loc_from_full_to_swa( - page_indices + if self.kv_index_translator.reads_are_translated: + # The translator writes kernel-facing page ids straight into + # the captured tables; `sliding_window_out` fills the swa twin + # in the same pass. + self.kv_index_translator.fill_read_table( + out=metadata.page_table, + sliding_window_out=( + metadata.swa_page_table + if self.use_sliding_window_kv_pool + and metadata.swa_page_table is not None + else None + ), + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_len_delta=self._spec_read_seq_len_delta( + forward_mode, spec_info + ), ) - metadata.swa_page_table[:, :max_seq_pages].copy_( - swa_page_indices // self.page_size + else: + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.draft_extend_metadata["strided_indices"][:max_seq_pages], + ] + if ( + self.use_sliding_window_kv_pool + and metadata.swa_page_table is not None + ): + swa_page_indices = ( + self.token_to_kv_pool.translate_loc_from_full_to_swa( + page_indices + ) + ) + metadata.swa_page_table[:, :max_seq_pages].copy_( + swa_page_indices // self.page_size + ) + metadata.page_table[:, :max_seq_pages].copy_( + page_indices // self.page_size ) - metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size) else: raise ValueError( diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 22d0a9b915..d9765566cd 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -1840,10 +1840,6 @@ class FlashInferIndicesUpdaterPrefill: self.kv_indptr = attn_backend.kv_indptr self.kv_last_page_len = attn_backend.kv_last_page_len self.qo_indptr = attn_backend.qo_indptr - # Kept ONLY for the spec-info branches (generate_attn_arg_prefill), - # which are static-pool-only: unified memory asserts spec off. The - # normal builders source from the per-batch KVIndexTable. - self.req_to_token = model_runner.req_to_token_pool.req_to_token self._swa_kv_pool = attn_backend._swa_kv_pool self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged @@ -2202,20 +2198,22 @@ class FlashInferIndicesUpdaterPrefill: if spec_info.spec_input_type == SpecInputType.DFLASH_VERIFY: kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - paged_kernel_lens, - paged_kernel_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=paged_kernel_lens, + paged_kernel_lens_sum=paged_kernel_lens_sum, + translator=translator, + sliding_window=use_swa_source, kv_start_idx=kv_start_idx, ) ) else: kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - paged_kernel_lens, - paged_kernel_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=paged_kernel_lens, + paged_kernel_lens_sum=paged_kernel_lens_sum, + translator=translator, + sliding_window=use_swa_source, ) ) diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 51188d0677..64023c962d 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -992,10 +992,6 @@ class FlashInferMLAIndicesUpdaterPrefill: # Buffers and wrappers self.kv_indptr = attn_backend.kv_indptr self.qo_indptr = attn_backend.qo_indptr - # Kept ONLY for the spec-info branch (generate_attn_arg_prefill), which - # is static-pool-only: unified memory asserts spec off. The normal - # builder reads req_to_token through the translator. - self.req_to_token = model_runner.req_to_token_pool.req_to_token self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged def update( @@ -1085,10 +1081,10 @@ class FlashInferMLAIndicesUpdaterPrefill: elif fast_verify_plan_kwargs is not None: kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - paged_kernel_lens, - paged_kernel_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=paged_kernel_lens, + paged_kernel_lens_sum=paged_kernel_lens_sum, + translator=self.attn_backend.kv_index_translator, kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"], ) ) @@ -1097,10 +1093,10 @@ class FlashInferMLAIndicesUpdaterPrefill: # TODO: Support topk > 1 with custom mask kv_indices, kv_indptr, qo_indptr, custom_mask = ( spec_info.generate_attn_arg_prefill( - req_pool_indices, - paged_kernel_lens, - paged_kernel_lens_sum, - self.req_to_token, + req_pool_indices=req_pool_indices, + paged_kernel_lens=paged_kernel_lens, + paged_kernel_lens_sum=paged_kernel_lens_sum, + translator=self.attn_backend.kv_index_translator, ) ) diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index 4b848cb099..26f99b997b 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -163,7 +163,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): if forward_batch.forward_mode.is_decode_or_idle(): max_seqlen_pad = triton.cdiv(eager_max_k, PAGE_SIZE) block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) - if self.kv_index_translator.is_translating: + if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE self.kv_index_translator.fill_read_table( out=block_kv_indices, @@ -198,17 +198,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend): max_seqlen_pad = triton.cdiv(eager_max_k + self.num_draft_tokens, PAGE_SIZE) block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) - create_flashmla_kv_indices_triton[ - (bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE)) - ]( - self.req_to_token, - forward_batch.req_pool_indices, - seq_lens, - None, - block_kv_indices, - self.req_to_token.stride(0), - block_kv_indices.stride(0), - ) + if self.kv_index_translator.reads_are_translated: + assert self.page_size == PAGE_SIZE + self.kv_index_translator.fill_read_table( + out=block_kv_indices, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=seq_lens, + ) + else: + create_flashmla_kv_indices_triton[ + (bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE)) + ]( + self.req_to_token, + forward_batch.req_pool_indices, + seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + block_kv_indices.stride(0), + ) mla_metadata, num_splits = get_mla_metadata( seq_lens.to(torch.int32), self.num_draft_tokens * self.num_q_heads, @@ -230,17 +238,25 @@ class FlashMLABackend(FlashInferMLAAttnBackend): max_seqlen_pad = triton.cdiv(eager_max_k + window, PAGE_SIZE) block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad) - create_flashmla_kv_indices_triton[ - (bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE)) - ]( - self.req_to_token, - forward_batch.req_pool_indices, - seq_lens_k, - None, - block_kv_indices, - self.req_to_token.stride(0), - block_kv_indices.stride(0), - ) + if self.kv_index_translator.reads_are_translated: + assert self.page_size == PAGE_SIZE + self.kv_index_translator.fill_read_table( + out=block_kv_indices, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=seq_lens_k, + ) + else: + create_flashmla_kv_indices_triton[ + (bs, get_num_kv_index_blocks_flashmla(max_seqlen_pad, PAGE_SIZE)) + ]( + self.req_to_token, + forward_batch.req_pool_indices, + seq_lens_k, + None, + block_kv_indices, + self.req_to_token.stride(0), + block_kv_indices.stride(0), + ) mla_metadata, num_splits = get_mla_metadata( seq_lens_k, window * self.num_q_heads, @@ -337,7 +353,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend): else: max_seqlen_pad = self.cuda_graph_kv_indices.shape[1] - if self.kv_index_translator.is_translating: + if self.kv_index_translator.reads_are_translated: assert self.page_size == PAGE_SIZE self.kv_index_translator.fill_read_table( out=self.cuda_graph_kv_indices, diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index c4461e2d7b..99691e095d 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -257,6 +257,7 @@ class KVIndexTranslator: seq_lens: torch.Tensor, max_pages: Optional[int] = None, into: Optional[KVReadTables] = None, + seq_len_delta: int = 0, ) -> KVIndexTable: """The one per-batch entry point. @@ -265,6 +266,10 @@ class KVIndexTranslator: returns the table WHOLE, so a caller needing a stable pointer (a captured graph bakes it) passes its own tables in ``into``; ``into=None`` allocates of width ``max_pages`` instead. + + ``seq_len_delta`` widens every row's live prefix -- the whole-sequence + verify contract (draft KV read back from the pool). An eager caller's + ``max_pages`` must already cover the delta. """ if not self.is_translating or self.defer_read_translate: return KVIndexTable( @@ -308,6 +313,7 @@ class KVIndexTranslator: page_size=self.page_size, max_pages=width, out=out_full, + seq_len_delta=seq_len_delta, ) if out_swa is not None: build_kv_read_table( @@ -318,6 +324,7 @@ class KVIndexTranslator: page_size=self.page_size, max_pages=width, out=out_swa, + seq_len_delta=seq_len_delta, ) return KVIndexTable( ids=out_full, @@ -335,6 +342,7 @@ class KVIndexTranslator: req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, sliding_window_out: Optional[torch.Tensor] = None, + seq_len_delta: int = 0, ) -> None: """`build_index_table(into=...)` for a caller that owns a bare block table rather than a KVReadTables: the page-table consumers read that @@ -359,6 +367,7 @@ class KVIndexTranslator: req_pool_indices=req_pool_indices, seq_lens=seq_lens, into=KVReadTables(full=out, sliding_window=sliding_window_out), + seq_len_delta=seq_len_delta, ) def index_table_for_batch(self, forward_batch) -> KVIndexTable: @@ -404,6 +413,38 @@ class KVIndexTranslator: """ return self._full_v2p_table + def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable: + """Whole-sequence-verify view: every row's live prefix widened by + `seq_len_delta` columns (the drafts are read back from the pool). + Not memoized -- verify is one consumer, and the batch's memoized + prefix table stays valid for the others. + + The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch + carries `seq_lens_cpu` already expanded by the window and keeps the + live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded + lens would count the window twice. The width never exceeds the + `req_to_token` row it is read from.""" + max_pages = None + if self.is_translating: + row_pages = -(-self.req_to_token.shape[1] // self.page_size) + live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None) + slc = live if live is not None else forward_batch.seq_lens_cpu + if ( + forward_batch.seq_lens_sum is not None + and slc is not None + and slc.numel() > 0 + ): + max_seq = int(slc.max()) + seq_len_delta + max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages) + else: + max_pages = row_pages + return self.build_index_table( + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens, + max_pages=max_pages, + seq_len_delta=seq_len_delta, + ) + def bind_and_verify_backends(self, backends) -> None: """Boot: make every reachable backend carry THIS translator. diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 86c1c1df02..170cee2efe 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -349,6 +349,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): layer_id=self.layer_id, forward_batch=forward_batch, mup_vector=self.mup_vector, + use_triton_causal_conv=forward_batch.forward_mode.is_target_verify(), ) mamba_hidden_states = mamba_hidden_states * self.ssm_out_multiplier diff --git a/python/sglang/srt/speculative/dflash_info.py b/python/sglang/srt/speculative/dflash_info.py index 9427ed944c..a004445e52 100644 --- a/python/sglang/srt/speculative/dflash_info.py +++ b/python/sglang/srt/speculative/dflash_info.py @@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, ForwardBatch, @@ -128,15 +128,20 @@ class DFlashVerifyInput(SpecInput): def generate_attn_arg_prefill( self, + *, req_pool_indices: torch.Tensor, paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, - req_to_token: torch.Tensor, + translator: KVIndexTranslator, kv_start_idx: Optional[torch.Tensor] = None, kv_indices_buf: Optional[torch.Tensor] = None, + sliding_window: bool = False, ): + """CSR verify args, gathered straight into the packed stream. The lens + are widened here and handed to the translator, so nothing materializes + a ``[bs, max_pages]`` rectangle to repack from.""" device = req_pool_indices.device - bs = len(req_pool_indices) + bs = req_pool_indices.numel() layout = self.ragged_verify_layout if layout is not None and layout.bs != bs: @@ -174,14 +179,14 @@ class DFlashVerifyInput(SpecInput): dtype=torch.int32, device=device, ) - create_flashinfer_kv_indices_triton[(bs,)]( - req_to_token, - req_pool_indices, - paged_kernel_lens, - cum_kv_seq_len, - kv_start_idx, - kv_indices, - req_to_token.size(1), + translator.fill_packed_read_stream( + req_pool_indices=req_pool_indices, + seq_lens=paged_kernel_lens, + indptr=cum_kv_seq_len, + total_tokens=paged_kernel_lens_sum + kv_indices_extra, + out=kv_indices, + kv_start_idx=kv_start_idx, + sliding_window=sliding_window, ) mask = self.custom_mask if mask is not None: diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index 1e2f5e5b4b..4fd02f6282 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -4,7 +4,7 @@ from typing import Callable, List, Optional import torch -from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode from sglang.srt.runtime_context import get_spec from sglang.srt.speculative.spec_info import SpecInput, SpecInputType @@ -85,13 +85,18 @@ class EagleVerifyInput(SpecInput): def generate_attn_arg_prefill( self, + *, req_pool_indices: torch.Tensor, paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, - req_to_token: torch.Tensor, + translator: KVIndexTranslator, + sliding_window: bool = False, ): + """CSR verify args, gathered straight into the packed stream. The lens + are widened here and handed to the translator, so nothing materializes + a ``[bs, max_pages]`` rectangle to repack from.""" device = req_pool_indices.device - batch_size = len(req_pool_indices) + batch_size = req_pool_indices.numel() qo_indptr = torch.arange( 0, (1 + batch_size) * self.draft_token_num, @@ -106,19 +111,15 @@ class EagleVerifyInput(SpecInput): paged_kernel_lens = paged_kernel_lens + self.draft_token_num cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0) - kv_indices = torch.empty( - paged_kernel_lens_sum + self.draft_token_num * batch_size, - dtype=torch.int32, - device=device, - ) - create_flashinfer_kv_indices_triton[(batch_size,)]( - req_to_token, - req_pool_indices, - paged_kernel_lens, - cum_kv_seq_len, - None, - kv_indices, - req_to_token.size(1), + total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size + kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device) + translator.fill_packed_read_stream( + req_pool_indices=req_pool_indices, + seq_lens=paged_kernel_lens, + indptr=cum_kv_seq_len, + total_tokens=total_tokens, + out=kv_indices, + sliding_window=sliding_window, ) mask_numel = ( paged_kernel_lens_sum * self.draft_token_num @@ -396,11 +397,15 @@ class EagleDraftExtendInput(SpecInput): def generate_attn_arg_prefill( self, + *, req_pool_indices: torch.Tensor, paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: Optional[int], - req_to_token: torch.Tensor, + translator: KVIndexTranslator, + sliding_window: bool = False, ): + """Draft-extend CSR args. The lens already include the window, so + unlike verify nothing is widened here.""" device = req_pool_indices.device bs = self.num_correct_drafts.numel() # Constant num_tokens_per_req qo layout (required for cuda-graph capture). @@ -421,13 +426,12 @@ class EagleDraftExtendInput(SpecInput): paged_kernel_lens_sum, dtype=torch.int32, device=device ) - create_flashinfer_kv_indices_triton[(bs,)]( - req_to_token, - req_pool_indices, - paged_kernel_lens, - cum_kv_seq_len, - None, - kv_indices, - req_to_token.size(1), + translator.fill_packed_read_stream( + req_pool_indices=req_pool_indices, + seq_lens=paged_kernel_lens, + indptr=cum_kv_seq_len, + total_tokens=paged_kernel_lens_sum, + out=kv_indices, + sliding_window=sliding_window, ) return kv_indices, cum_kv_seq_len, qo_indptr, None diff --git a/python/sglang/srt/speculative/ngram_info.py b/python/sglang/srt/speculative/ngram_info.py index 6cb9ebe0f7..ce5d7cbabe 100644 --- a/python/sglang/srt/speculative/ngram_info.py +++ b/python/sglang/srt/speculative/ngram_info.py @@ -4,7 +4,7 @@ from typing import List, Optional import torch -from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator from sglang.srt.speculative.spec_info import SpecInput, SpecInputType @@ -58,12 +58,17 @@ class NgramVerifyInput(SpecInput): def generate_attn_arg_prefill( self, + *, req_pool_indices: torch.Tensor, paged_kernel_lens: torch.Tensor, paged_kernel_lens_sum: int, - req_to_token: torch.Tensor, + translator: KVIndexTranslator, + sliding_window: bool = False, ): - bs = len(req_pool_indices) + """CSR verify args, gathered straight into the packed stream. The lens + are widened here and handed to the translator, so nothing materializes + a ``[bs, max_pages]`` rectangle to repack from.""" + bs = req_pool_indices.numel() cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device) @@ -75,20 +80,16 @@ class NgramVerifyInput(SpecInput): * self.draft_token_num ) - kv_indices = torch.empty( - paged_kernel_lens_sum + self.draft_token_num * bs, - dtype=torch.int32, - device=self.device, - ) + total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs + kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device) - create_flashinfer_kv_indices_triton[(bs,)]( - req_to_token, - req_pool_indices, - paged_kernel_lens, - cum_kv_seq_len, - None, - kv_indices, - req_to_token.size(1), + translator.fill_packed_read_stream( + req_pool_indices=req_pool_indices, + seq_lens=paged_kernel_lens, + indptr=cum_kv_seq_len, + total_tokens=total_tokens, + out=kv_indices, + sliding_window=sliding_window, ) # Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests. diff --git a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py index ea750ff195..d9f8fb4e58 100644 --- a/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py +++ b/test/registered/unit/layers/attention/test_flashattention_graph_metadata.py @@ -88,5 +88,65 @@ class TestFlashAttentionGraphMetadata(CustomTestCase): self.assertEqual(backend.forward_metadata.max_seq_len_k, 16) +class TestSpecReadSeqLenDelta(CustomTestCase): + """The mode -> seq_len_delta map IS the widening the whole-sequence spec + reads apply to cache_seqlens; a drift silently truncates the verify or + draft tail of the translated read table (the metadata builders widen + cache_seqlens while the source fill stays prefix-only).""" + + def _backend(self, *, topk=1, num_steps=3, step_id=1, num_draft=4): + b = FlashAttentionBackend.__new__(FlashAttentionBackend) + b.topk = topk + b.speculative_num_steps = num_steps + b.speculative_step_id = step_id + b.speculative_num_draft_tokens = num_draft + return b + + def test_mode_to_delta_map(self): + b = self._backend() + spec = object() + # Verify reads [prefix + drafts]; draft decode step i reads + # [prefix + i + 1] (both write the drafts into the pool first). + self.assertEqual(b._spec_read_seq_len_delta(ForwardMode.TARGET_VERIFY, spec), 4) + self.assertEqual(b._spec_read_seq_len_delta(ForwardMode.DECODE, spec), 2) + # Prefix-only shapes stay un-widened. + self.assertEqual( + b._spec_read_seq_len_delta(ForwardMode.DRAFT_EXTEND_V2, spec), 0 + ) + self.assertEqual(b._spec_read_seq_len_delta(ForwardMode.DECODE, None), 0) + self.assertEqual(b._spec_read_seq_len_delta(ForwardMode.EXTEND, spec), 0) + # Draft-extend's idle batch carries no live draft chain. + idle = self._backend(num_steps=0) + self.assertEqual(idle._spec_read_seq_len_delta(ForwardMode.IDLE, spec), 0) + # topk>1 reads the drafts via the expand metadata - prefix only. + tree = self._backend(topk=2) + self.assertEqual( + tree._spec_read_seq_len_delta(ForwardMode.TARGET_VERIFY, spec), 0 + ) + self.assertEqual(tree._spec_read_seq_len_delta(ForwardMode.DECODE, spec), 0) + + +class TestDraftExtendInGraph(CustomTestCase): + """The draft-extend cuda-graph runners record + `init_forward_metadata_in_graph` unconditionally. Under translation fa3 + rebuilds the draft-extend tables out of graph on every replay, so the + in-graph gather -- raw req_to_token, i.e. virtual ids -- must not be + recorded, or each replay overwrites the translated tables.""" + + def _run(self, *, translating): + b = FlashAttentionBackend.__new__(FlashAttentionBackend) + b.kv_index_translator = SimpleNamespace(is_translating=translating) + b.draft_extend_metadata = {} # any metadata access raises KeyError + fb = SimpleNamespace(forward_mode=ForwardMode.DRAFT_EXTEND_V2, batch_size=2) + b.init_forward_metadata_in_graph(fb) + + def test_translating_backend_records_nothing(self): + self._run(translating=True) + + def test_static_backend_still_builds_in_graph(self): + with self.assertRaises(KeyError): + self._run(translating=False) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/attention/test_verify_widening_is_wired.py b/test/registered/unit/layers/attention/test_verify_widening_is_wired.py new file mode 100644 index 0000000000..7c66f92a85 --- /dev/null +++ b/test/registered/unit/layers/attention/test_verify_widening_is_wired.py @@ -0,0 +1,208 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A widened verify table must reach the kernel that reads past seq_lens. + +A whole-sequence verify reads `[committed prefix | drafts]` back out of the +pool, so under the unified pool the translated page table a verify kernel +reads has to cover `seq_lens + num_draft_tokens` columns per row. A table +filled only to `seq_lens` leaves the draft tail stale: invisible at +page_size 256, where the un-widened `ceil(seq/ps)` pages still cover a few +draft tokens, and at page_size 1 observed as accept length collapsing +(6.20 -> 1.37) or wrong tokens (gsm8k 0.905 -> 0.730). + +These tests drive fa3's two verify builds -- the eager one and the captured +one a cuda-graph replay runs -- through a translating KV-index translator +over a non-identity virtual->physical page map, and check the page table the +kernel reads: every row translated, and filled through the draft tail. + + python -m pytest test/registered/unit/layers/attention/test_verify_widening_is_wired.py -v +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.attention.flashattention_backend import ( + FlashAttentionBackend, + FlashAttentionMetadata, +) +from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") + +_DEV = "cuda" +_NUM_DRAFT = 4 +_MAX_CTX = 128 + + +def _translator(req_to_token, page_size, n_pages): + """A translating translator over a fixed, non-identity page map. + + The allocator wiring is irrelevant here; what matters is that the reads go + through the real table builder and kernel with a v2p map that a skipped or + truncated translation cannot accidentally satisfy.""" + g = torch.Generator(device="cpu").manual_seed(7 + page_size) + v2p = torch.randperm(n_pages, generator=g).to(torch.int64) + v2p[0] = 0 # page 0 is the reserved sink + t = KVIndexTranslator.__new__(KVIndexTranslator) + t.req_to_token = req_to_token + t.page_size = page_size + t.device = _DEV + t.is_translating = True + t.defer_read_translate = False + t._capture_page_size = page_size + t._full_v2p_table = v2p.to(_DEV) + t._swa_v2p_table = None + t._rows = torch.arange(req_to_token.shape[0], dtype=torch.int64, device=_DEV) + t._index_table_memo = None + return t + + +def _batch(page_size, n_pages): + """Three requests with distinct, non-monotonic virtual page runs; each row + holds its prefix plus the draft window (verify writes the drafts first).""" + g = torch.Generator(device="cpu").manual_seed(11 + page_size) + live = torch.tensor([page_size + 1, 3 * page_size, 1], dtype=torch.int64) + bs = live.numel() + req_to_token = torch.zeros((bs, _MAX_CTX), dtype=torch.int32) + pages = torch.randperm(n_pages - 1, generator=g)[: bs * 8] + 1 + for r in range(bs): + run = pages[r * 8 : r * 8 + 8] + toks = (run[:, None] * page_size + torch.arange(page_size)[None, :]).reshape(-1) + n = int(live[r]) + _NUM_DRAFT + req_to_token[r, :n] = toks[:n].to(torch.int32) + return req_to_token.to(_DEV), live + + +def _expected(req_to_token, live, v2p, page_size): + """Per row, the translated pages covering `live + num_draft_tokens`.""" + rows = [] + for r in range(live.numel()): + n_pages = -(-(int(live[r]) + _NUM_DRAFT) // page_size) + toks = req_to_token[r, torch.arange(n_pages, device=_DEV) * page_size].long() + rows.append(v2p[toks // page_size].to(torch.int32).cpu()) + return rows + + +def _backend(translator, req_to_token, page_size): + b = FlashAttentionBackend.__new__(FlashAttentionBackend) + b.kv_index_translator = translator + b.req_to_token = req_to_token + b.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token) + b.page_size = page_size + b.max_context_len = _MAX_CTX + b.max_num_pages = -(-_MAX_CTX // page_size) + b.topk = 1 + b.speculative_num_draft_tokens = _NUM_DRAFT + b.speculative_num_steps = 3 + b.speculative_step_id = 0 + b.has_swa = False + b.use_sliding_window_kv_pool = False + b.is_prefill_aware_swa = False + b.token_to_kv_pool = None + b._kv_shard_pool = None + b._compute_scheduler_metadata = lambda *_: None + b._maybe_init_local_attn_metadata = lambda *_: None + return b + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestFa3VerifyReadsTheDraftTail(CustomTestCase): + N_PAGES = 64 + + def _check(self, page_table, req_to_token, live, translator, page_size): + want = _expected(req_to_token, live, translator._full_v2p_table, page_size) + got = page_table.cpu() + for r, row in enumerate(want): + self.assertTrue( + torch.equal(got[r, : row.numel()], row), + f"ps={page_size} row {r}: page table does not cover the draft " + f"tail:\ngot ={got[r, : row.numel()]}\nwant={row}", + ) + + def test_captured_verify_build(self): + """The captured build a cuda-graph replay runs.""" + for page_size in (1, 4): + req_to_token, live = _batch(page_size, self.N_PAGES) + translator = _translator(req_to_token, page_size, self.N_PAGES) + b = _backend(translator, req_to_token, page_size) + bs = live.numel() + metadata = FlashAttentionMetadata() + metadata.cache_seqlens_int32 = torch.zeros( + bs, dtype=torch.int32, device=_DEV + ) + metadata.cu_seqlens_k = torch.zeros(bs + 1, dtype=torch.int32, device=_DEV) + metadata.page_table = torch.zeros( + (bs, b.max_num_pages), dtype=torch.int32, device=_DEV + ) + metadata.swa_page_table = None + b.target_verify_metadata = {bs: metadata} + b._apply_cuda_graph_metadata( + bs, + torch.arange(bs, device=_DEV), + live.to(_DEV), + None, + ForwardMode.TARGET_VERIFY, + SimpleNamespace(ragged_verify_layout=None), + live, + ) + self.assertTrue( + torch.equal( + metadata.cache_seqlens_int32.cpu(), (live + _NUM_DRAFT).int() + ) + ) + self._check(metadata.page_table, req_to_token, live, translator, page_size) + + def test_eager_verify_build(self): + """The eager build, including a DSPARK-style batch whose host lens are + already expanded by the window.""" + for page_size in (1, 4): + req_to_token, live = _batch(page_size, self.N_PAGES) + translator = _translator(req_to_token, page_size, self.N_PAGES) + b = _backend(translator, req_to_token, page_size) + bs = live.numel() + for host_lens, spec_info in ( + (live, SimpleNamespace(ragged_verify_layout=None)), + ( + live + _NUM_DRAFT, + SimpleNamespace(ragged_verify_layout=None, live_seq_lens_cpu=live), + ), + ): + fb = SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=bs, + seq_lens=live.to(_DEV), + seq_lens_cpu=host_lens, + seq_lens_sum=int(host_lens.sum()), + req_pool_indices=torch.arange(bs, device=_DEV), + spec_info=spec_info, + encoder_lens=None, + out_cache_loc=None, + ) + b.init_forward_metadata(fb) + self._check( + b.forward_metadata.page_table, + req_to_token, + live, + translator, + page_size, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_kv_index_translator.py b/test/registered/unit/mem_cache/test_kv_index_translator.py index d2db0cad35..da57ee82b6 100644 --- a/test/registered/unit/mem_cache/test_kv_index_translator.py +++ b/test/registered/unit/mem_cache/test_kv_index_translator.py @@ -506,11 +506,13 @@ class _FakeForwardBatch: seq_lens_cpu=None, out_cache_loc=None, seq_lens_sum=-1, + spec_info=None, ): self.req_pool_indices = req_pool_indices self.seq_lens = seq_lens self.seq_lens_cpu = seq_lens_cpu self.out_cache_loc = out_cache_loc + self.spec_info = spec_info self.seq_lens_sum = ( (None if seq_lens is None else int(seq_lens.sum())) if seq_lens_sum == -1 @@ -518,6 +520,63 @@ class _FakeForwardBatch: ) +class TestWidenedTable(unittest.TestCase): + """`widened_index_table`: the whole-sequence verify view, `seq_len_delta` + columns past every row's live prefix.""" + + def setUp(self): + # The code under test reads its config from the bags. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="tokenizer") + + def _check(self, view, req_to_token, rows, lens, allocator, ps, width): + self.assertEqual(tuple(view.ids.shape), (rows.numel(), width)) + want = _reference_table( + req_to_token, rows, lens, allocator.full_v2p_page_table, ps, width + ) + self.assertTrue(torch.equal(view.ids, want), f"ps={ps}") + + def test_width_comes_from_the_live_prefix(self): + """A DSPARK verify batch carries `seq_lens_cpu` already expanded by the + window and the live lens on `spec_info.live_seq_lens_cpu`; widening the + expanded lens would count the window twice.""" + window = 3 + for ps in (1, 4): + allocator = _build_composite(ps) + req_to_token, rows, live = _alloc_and_fill( + allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1] + ) + src = _make_source(allocator, req_to_token, ps) + fb = _FakeForwardBatch( + req_pool_indices=rows, + seq_lens=live, + seq_lens_cpu=live + window, + spec_info=SimpleNamespace(live_seq_lens_cpu=live), + ) + view = src.widened_index_table(fb, seq_len_delta=window) + width = -(-(int(live.max()) + window) // ps) + self._check(view, req_to_token, rows, live + window, allocator, ps, width) + + def test_width_never_exceeds_the_request_row(self): + """Near the context limit the widened host bound can pass the end of + the `req_to_token` row; the table stops at the row instead of tripping + the read-table width assert.""" + window = 6 + ps = 1 + allocator = _build_composite(ps) + req_to_token, rows, live = _alloc_and_fill(allocator, ps, lens=[6, 1]) + row_pages = req_to_token.shape[1] // ps + src = _make_source(allocator, req_to_token, ps) + # No live lens on the spec info: the expanded mirror is all there is. + fb = _FakeForwardBatch( + req_pool_indices=rows, seq_lens=live, seq_lens_cpu=live + window + ) + self.assertGreater(int(live.max()) + 2 * window, row_pages) + view = src.widened_index_table(fb, seq_len_delta=window) + self._check(view, req_to_token, rows, live + window, allocator, ps, row_pages) + + class TestViewMemo(unittest.TestCase): """The eager view is memoized ON THE SOURCE in a single slot keyed by batch identity, so per-batch state stays out of the ForwardBatch while one diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py index af3fa742f1..25e878922b 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py +++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py @@ -145,6 +145,77 @@ class TestBlockTable(unittest.TestCase): v2p[0] = 0 # page 0 is the reserved sink return req_to_token, req_pool_indices, seq_lens, v2p + def test_seq_len_delta_matches_widened_lens(self): + """The kernel's ``seq_len_delta`` equals building with ``seq_lens + k`` + (the two spellings of verify widening), and genuinely widens: the + widened columns overwrite the -1 sentinel the plain build leaves.""" + from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table + + for page_size in (1, 32): + rt, rpi, sl, v2p = self._make_batch(page_size) + delta = page_size + 1 + max_pages = int((sl.max().item() + delta + page_size - 1) // page_size) + + def fill(seq_lens, seq_len_delta): + out = torch.full( + (rpi.shape[0], max_pages), -1, dtype=torch.int32, device=_DEV + ) + build_kv_read_table( + req_to_token=rt, + req_pool_indices=rpi, + seq_lens=seq_lens.to(torch.int64), + v2p=v2p, + page_size=page_size, + max_pages=max_pages, + out=out, + seq_len_delta=seq_len_delta, + ) + return out + + widened = fill(sl, delta) + by_lens = fill(sl + delta, 0) + plain = fill(sl, 0) + self.assertTrue(torch.equal(widened, by_lens), f"ps={page_size}") + self.assertFalse(torch.equal(widened, plain), f"ps={page_size}") + + def test_rows_stop_at_the_table_width(self): + """A widened window longer than the table's row (a verify near the + context limit) is cut at ``max_pages`` instead of running into the + next row -- another request's page table -- or past the buffer. The + CUDA kernel must agree with the CPU path, which clamps per row.""" + from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table + + for page_size in (1, 32): + rt, rpi, sl, v2p = self._make_batch(page_size) + bs = rpi.shape[0] + delta = 2 * page_size + max_pages = 3 + n_pages = (sl + delta + page_size - 1) // page_size + self.assertTrue(bool((n_pages > max_pages).any()), "no row overflows") + + def fill(device): + # One guard row past the table catches a spill off the end. + out = torch.full( + (bs + 1, max_pages), -1, dtype=torch.int32, device=device + ) + build_kv_read_table( + req_to_token=rt.to(device), + req_pool_indices=rpi.to(device), + seq_lens=sl.to(device=device, dtype=torch.int64), + v2p=v2p.to(device), + page_size=page_size, + max_pages=max_pages, + out=out[:bs], + seq_len_delta=delta, + ) + return out.cpu() + + gpu, cpu = fill(_DEV), fill("cpu") + self.assertTrue(torch.all(gpu[bs] == -1), f"ps={page_size}: {gpu}") + self.assertTrue( + torch.equal(gpu, cpu), f"ps={page_size}:\ngpu={gpu}\ncpu={cpu}" + ) + def test_static_kernel_matches_reference(self): """The stripped (id-space-free) flashmla kernel is byte-identical to the plain token//ps reference -- guards the v2p-arg removal itself.""" diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py new file mode 100644 index 0000000000..f5e9d32685 --- /dev/null +++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py @@ -0,0 +1,163 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""`--enable-unified-memory` speculative-decoding allow-list. + +One audited arm today: DSPARK, a chain draft whose draft KV lives in a pool of +its own. Its verify runs on the backends whose spec paths build their read +tables through the KV-index translator -- the MLA verify family plus +`flashmla` / `flashinfer` / `fa3`. Everything else stays refused until its +verify id rails are audited. + +Pinned so no arm silently widens to an unaudited algorithm, tree shape, or +backend. The backend set in particular is a claim about code that exists: a +backend listed here MUST have a translated spec path, or a unified run on it +reads the pool with virtual ids and silently returns wrong tokens. + + python -m pytest test/registered/unit/server_args/test_unified_memory_spec_gate.py -v +""" + +import unittest +from types import SimpleNamespace + +import msgspec + +from sglang.srt.arg_groups.kv_cache_hook import handle_unified_memory_pool +from sglang.srt.configs.model_config import AttentionArch +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _accepts( + algorithm: str | None, + *, + topk: int | None = 1, + backend: str | None = "triton", + draft_backend: str | None = None, + dcp_size: int = 1, + mla: bool = False, +) -> bool: + """Run just `handle_unified_memory_pool` against a minimal stand-in. + + ServerArgs' real constructor pulls in a model config; this exercises the + single handler under test with the fields it reads (disaggregation off, so + only the speculative / cache / dcp / cuda-graph checks run). ServerArgs is + a msgspec Struct, so a field is restated with `force_setattr`, which the + Struct's own `__setattr__` guard allows and `object.__setattr__` does not. + """ + sa = ServerArgs(model_path="dummy") + for name, value in { + "enable_unified_memory": True, + "disaggregation_mode": "null", + "speculative_algorithm": algorithm, + "speculative_eagle_topk": topk, + "speculative_draft_attention_backend": draft_backend, + "enable_hierarchical_cache": False, + "enable_lmcache": False, + "dcp_size": dcp_size, + "cuda_graph_config": None, + "attention_backend": backend, + "prefill_attention_backend": None, + "decode_attention_backend": None, + }.items(): + msgspec.structs.force_setattr(sa, name, value) + sa._model_config = SimpleNamespace( + is_hybrid_swa=not mla, + attention_arch=AttentionArch.MLA if mla else AttentionArch.MHA, + ) + try: + handle_unified_memory_pool(sa) + return True + except AssertionError: + return False + + +class TestUnifiedMemorySpecGate(unittest.TestCase): + # Backends whose speculative verify path builds through the translator. + AUDITED_BACKENDS = ( + "triton", + "trtllm_mla", + "cutedsl_mla", + "tokenspeed_mla", + "flashmla", + "flashinfer", + "fa3", + ) + # Algorithms with no audited unified-pool verify rails. + UNAUDITED_ALGORITHMS = ("EAGLE", "EAGLE3", "DFLASH", "NGRAM", "STANDALONE") + + def test_dspark_admitted_on_every_audited_backend(self): + """Each entry is a claim that the backend's spec verify path + translates. Adding one here without the code is how a unified run + silently reads the pool with virtual ids.""" + for backend in self.AUDITED_BACKENDS: + self.assertTrue( + _accepts("DSPARK", backend=backend), + f"DSPARK should pass on verify-audited backend {backend}", + ) + + def test_dspark_refused_on_unaudited_backends(self): + """fa4 and trtllm_mha have no translated spec verify path.""" + for backend in ("fa4", "trtllm_mha"): + self.assertFalse(_accepts("DSPARK", backend=backend)) + + def test_tree_drafting_refused(self): + """Tree verify needs a per-token move inside the TARGET pool, which + the unified pool's page-granular `move_kv_cache` cannot express. + DSPARK is the only admitted algorithm today, so it is the only one + that can demonstrate the rule; the check itself is unconditional, so + it keeps holding as further arms are admitted.""" + for topk in (2, 4, 8): + self.assertFalse(_accepts("DSPARK", topk=topk)) + # The shape knob must not disturb spec-off. + self.assertTrue(_accepts(None, topk=None)) + + def test_dspark_draft_backend_is_not_constrained(self): + """DSPARK's draft owns a KV pool indexed directly by virtual id, so + its translator is a passthrough and any draft backend reads correctly. + Model hooks declare one on the operator's behalf -- Kimi-Linear / + Kimi-K3 + DSPARK on SM100 declares `trtllm_mha` -- so refusing it + would refuse a configuration the operator never touched.""" + for draft_backend in (None, *self.AUDITED_BACKENDS, "fa4", "trtllm_mha"): + self.assertTrue( + _accepts("DSPARK", draft_backend=draft_backend), + f"DSPARK draft backend {draft_backend} should pass", + ) + + def test_dcp_keeps_flashinfer_off_the_verify_list(self): + """Under --dcp-size > 1 the read ids stay DCP-widened for the consumer + to finish; flashinfer's spec verify gathers its CSR args with no DCP + read translation, so it is refused there while the MLA verify family, + which builds its own DCP block table, still passes.""" + self.assertTrue(_accepts("DSPARK", backend="trtllm_mla", dcp_size=2, mla=True)) + self.assertFalse(_accepts("DSPARK", backend="flashinfer", dcp_size=2, mla=True)) + # Without DCP flashinfer stays admitted. + self.assertTrue(_accepts("DSPARK", backend="flashinfer", mla=True)) + + def test_spec_off_admitted(self): + """The gate constrains only speculative configurations; spec-off must + keep booting.""" + self.assertTrue(_accepts(None)) + + def test_unaudited_algorithms_refused(self): + """Every other algorithm stays out until its verify id rails are + audited -- on every backend.""" + for algorithm in self.UNAUDITED_ALGORITHMS: + for backend in ("triton", "fa3"): + self.assertFalse(_accepts(algorithm, backend=backend)) + + +if __name__ == "__main__": + unittest.main()