diff --git a/python/sglang/kernels/ops/attention/kv_indices.py b/python/sglang/kernels/ops/attention/kv_indices.py index 5779416ebd..132f0a1f9c 100644 --- a/python/sglang/kernels/ops/attention/kv_indices.py +++ b/python/sglang/kernels/ops/attention/kv_indices.py @@ -1,3 +1,5 @@ +from typing import Optional + import triton import triton.language as tl @@ -18,6 +20,41 @@ def kv_indices_num_token_blocks(table_width: int, base_programs: int) -> int: return max(1, min(cap, want)) +# Token-block parallelism pays only on long-context servers; below this the +# spec-decode index launches keep their historical grids. +SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768 + + +def spec_kv_index_token_blocks( + table_width: int, + kv_lens_sum: Optional[int], + batch_size: int, + base_programs: Optional[int] = None, + length_cap: Optional[int] = None, + context_len: Optional[int] = None, +) -> int: + """Token blocks per base program for a spec-decode KV-index launch. + + 1 below the long-context gate (``context_len``, default ``table_width``). + Otherwise sized from an upper bound on the longest live row — the launch + time is set by the longest row, and ``kv_lens_sum`` bounds it from above + while still a host int (tightened by ``length_cap`` and ``table_width``, + which also bound any single row). Any block count is correct; it only + changes the parallelism. + """ + gate = table_width if context_len is None else context_len + if gate < SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT or batch_size <= 0: + return 1 + if kv_lens_sum is None: + width = table_width + else: + width = min(kv_lens_sum, table_width) + if length_cap is not None: + width = min(width, length_cap) + programs = batch_size if base_programs is None else base_programs + return kv_indices_num_token_blocks(width, programs) + + @triton.jit def create_flashinfer_kv_indices_triton( req_to_token_ptr, # [max_batch, max_context_len] token table; at diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index d25aaf1ade..bbfe463bf6 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -23,7 +23,7 @@ from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, create_flashmla_kv_indices_triton, get_num_kv_index_blocks_flashmla, - kv_indices_num_token_blocks, + spec_kv_index_token_blocks, ) from sglang.kernels.ops.attention.utils import assert_buffer_fits from sglang.srt.layers.attention.base_attn_backend import AttentionBackend @@ -155,13 +155,6 @@ fast_mode = False intra_batch_mode = True if _use_mla_ps_kernel else False -# Token-block parallel KV-index building is enabled only where it pays: -# the speculative-decoding paths (target_verify / draft_extend / draft -# decode) of long-context servers. Everything else keeps the historical -# one-program-per-request launch. -_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768 - - class WrapperDispatch(Enum): SLIDING_WINDOW = auto() CROSS_ATTENTION = auto() @@ -1968,9 +1961,12 @@ class AiterAttnBackend(AttentionBackend): return output, final_lse def _kv_index_blocks(self, bs: int) -> int: - if self.max_context_len < _KV_INDEX_BLOCKS_MIN_CONTEXT: - return 1 - return kv_indices_num_token_blocks(self.req_to_token.shape[1], bs) + return spec_kv_index_token_blocks( + self.req_to_token.shape[1], + kv_lens_sum=None, + batch_size=bs, + context_len=self.max_context_len, + ) def init_forward_metadata_out_graph( self, @@ -5125,12 +5121,12 @@ class AiterMultiStepDraftBackend: bs = self.topk * num_seqs seq_lens_sum = forward_batch.seq_lens_sum - num_token_blocks = ( - kv_indices_num_token_blocks( - self.pool_len, self.speculative_num_steps * num_seqs * self.topk - ) - if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT - else 1 + num_token_blocks = spec_kv_index_token_blocks( + self.pool_len, + kv_lens_sum=None, + batch_size=num_seqs, + base_programs=self.speculative_num_steps * num_seqs * self.topk, + context_len=self.max_context_len, ) src = self.kv_index_translator.read_source( forward_batch.kv_loc_plan, diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 168350a0eb..81aba85e02 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Union import torch from sglang.kernels.kernel_api_logging import debug_kernel_api +from sglang.kernels.ops.attention.kv_indices import spec_kv_index_token_blocks from sglang.kernels.ops.attention.utils import ( assert_buffer_fits, ) @@ -67,7 +68,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) - if envs.SGLANG_ENABLE_TORCH_COMPILE.get(): torch._logging.set_logs(dynamo=logging.ERROR) torch._dynamo.config.suppress_errors = True @@ -2512,6 +2512,21 @@ class FlashInferMultiStepDraftBackend: seq_lens_sum=seq_lens_sum, ) + # Blocks from the length sum (upper bound on the longest live row), + # capped by the draft window, not from the table width. pool_len is + # the req_to_token row width, the same table width the other call + # sites size from. + num_token_blocks = spec_kv_index_token_blocks( + table_width=self.pool_len, + kv_lens_sum=seq_lens_sum, + batch_size=num_seqs, + base_programs=self.speculative_num_steps * num_seqs * self.topk, + length_cap=( + self.draft_window_size + self.draft_sink_size + if self.draft_window_size > 0 + else None + ), + ) src = self.kv_index_translator.read_source( forward_batch.kv_loc_plan, req_pool_indices=forward_batch.req_pool_indices, @@ -2532,8 +2547,11 @@ class FlashInferMultiStepDraftBackend: if use_host_indptr else 1 ) + # One launch, the larger of the two tilings: the host-plan path never + # drops below its 512-token blocks. + num_token_blocks = max(num_token_blocks, num_blocks) self.generate_draft_decode_kv_indices[ - (self.speculative_num_steps * num_blocks, num_seqs, self.topk) + (self.speculative_num_steps * num_token_blocks, num_seqs, self.topk) ]( src.row_ids, src.ids, @@ -2553,7 +2571,14 @@ class FlashInferMultiStepDraftBackend: ENTRY_PAGE_SIZE=src.entry_page_size, v2p=src.v2p, TRANSLATE=src.v2p is not None, - NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0, + # A single token block without the host plan is the historical + # launch; NUM_STEPS=0 keeps its 128-wide program instead of the + # token-block specialization. + NUM_STEPS=( + self.speculative_num_steps + if num_token_blocks > 1 or use_host_indptr + else 0 + ), ) assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 821c8509cd..d9a6a9f0c6 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -69,6 +69,7 @@ import torch from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, + spec_kv_index_token_blocks, ) from sglang.kernels.ops.kvcache.kv_read_table import ( build_kv_read_table, @@ -538,14 +539,28 @@ class KVIndexTranslator: kv_start_idx: Optional[torch.Tensor] = None, kind: IdSpaceKind = IdSpaceKind.FULL, token_mapping: Optional[torch.Tensor] = None, + num_token_blocks: Optional[int] = 1, + kv_lens_sum: Optional[int] = None, ) -> bool: """Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans over; return whether they are physical. Packed from the plan's table once a table reader has had it built, else gathered and translated straight into ``out``. ``False`` means still-VIRTUAL full-attention ids (a static pool, or DCP) for the caller to finish; a static SWA pool can - fuse its full->swa table into the gather as ``token_mapping``.""" + fuse its full->swa table into the gather as ``token_mapping``. + ``num_token_blocks`` tiles each request's gather over that many + programs (any count is correct); ``None`` sizes it here with + ``spec_kv_index_token_blocks`` from ``kv_lens_sum``, the batch's + host-int length sum (an upper bound on the longest row; ``None`` + falls back to the table width). The packed-table build path has its + own parallelism and ignores the count.""" bs = int(seq_lens.numel()) + if num_token_blocks is None: + num_token_blocks = spec_kv_index_token_blocks( + table_width=self.req_to_token.size(1), + kv_lens_sum=kv_lens_sum, + batch_size=bs, + ) if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self): assert plan.is_read_by(self), ( "a translating reader must read through the plan of its own " @@ -576,7 +591,7 @@ class KVIndexTranslator: ) assert src.v2p is None, "pack_read_stream: a table-less translating read" assert token_mapping is None or not src.is_translated - create_flashinfer_kv_indices_triton[(bs,)]( + create_flashinfer_kv_indices_triton[(bs, num_token_blocks)]( src.ids, src.row_ids[:bs], seq_lens, @@ -585,6 +600,7 @@ class KVIndexTranslator: out, src.row_stride, ENTRY_PAGE_SIZE=src.entry_page_size, + TOKEN_BLOCK_PARALLEL=num_token_blocks > 1, token_mapping=token_mapping, ) return src.is_translated or token_mapping is not None diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index f9a0ae2b5a..99a3aba190 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -125,6 +125,11 @@ class EagleVerifyInput(SpecInput): indptr=cum_kv_seq_len, out=kv_indices, kind=kind, + # The translator tiles the gather over token blocks, sized from + # the length sum (an upper bound on the longest live row) while + # it is still a host int. + num_token_blocks=None, + kv_lens_sum=paged_kernel_lens_sum, ) mask_numel = ( paged_kernel_lens_sum * self.draft_token_num @@ -429,6 +434,10 @@ class EagleDraftExtendInput(SpecInput): cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=device) cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0) + # Captured while the length sum is still a host int (None -> the + # translator sizes from its table width). + host_kv_lens_sum = paged_kernel_lens_sum + if paged_kernel_lens_sum is None: paged_kernel_lens_sum = cum_kv_seq_len[-1] @@ -443,5 +452,7 @@ class EagleDraftExtendInput(SpecInput): indptr=cum_kv_seq_len, out=kv_indices, kind=kind, + num_token_blocks=None, + kv_lens_sum=host_kv_lens_sum, ) return kv_indices, cum_kv_seq_len, qo_indptr, None diff --git a/test/registered/kernels/ops/speculative/test_draft_decode_window.py b/test/registered/kernels/ops/speculative/test_draft_decode_window.py index 0617d295f7..b0694e2b75 100644 --- a/test/registered/kernels/ops/speculative/test_draft_decode_window.py +++ b/test/registered/kernels/ops/speculative/test_draft_decode_window.py @@ -107,10 +107,13 @@ def _make_inputs(seq_lens, num_steps, topk, page_size=1, pool_len=None): ) -def _run(io, window_size=0, sink_size=0): +def _run(io, window_size=0, sink_size=0, num_token_blocks=1): + """``num_token_blocks > 1`` is the long-context launch of the flashinfer / + aiter draft backends: ``NUM_STEPS`` splits each row over token blocks.""" kv_indices = io["kv_indices"].clone() kv_indptr = io["kv_indptr"].clone() - generate_draft_decode_kv_indices[(io["num_steps"], io["num_seqs"], io["topk"])]( + grid = (io["num_steps"] * num_token_blocks, io["num_seqs"], io["topk"]) + generate_draft_decode_kv_indices[grid]( io["req_pool_indices"], io["req_to_token"], io["seq_lens"], @@ -126,6 +129,7 @@ def _run(io, window_size=0, sink_size=0): io["page_size"], window_size, sink_size, + NUM_STEPS=io["num_steps"] if num_token_blocks > 1 else 0, ) torch.cuda.synchronize() return kv_indices, kv_indptr @@ -233,7 +237,7 @@ class TestDraftDecodeWindowKernel(unittest.TestCase): f"tree tokens wrong at step {step} (seq_len={seq_len})", ) - def _assert_slot_layout(self, io, window, sink): + def _assert_slot_layout(self, io, window, sink, num_token_blocks=1): """Every (request, topk) slot holds [sink + recent] + its own tree branch. Slots are located by the closed-form kv_indptr oracle, so this also pins @@ -242,7 +246,9 @@ class TestDraftDecodeWindowKernel(unittest.TestCase): covers the identity case too. """ cap = None if window == 0 else window + sink - kv_indices, indptr = _run(io, window_size=window, sink_size=sink) + kv_indices, indptr = _run( + io, window_size=window, sink_size=sink, num_token_blocks=num_token_blocks + ) expected_indptr = _expected_kv_indptr(io, cap=cap) self.assertTrue(torch.equal(indptr.cpu(), expected_indptr)) @@ -296,6 +302,34 @@ class TestDraftDecodeWindowKernel(unittest.TestCase): io = _make_inputs(seq_lens, num_steps, topk) self._assert_slot_layout(io, window, sink) + def test_token_block_grid_matches_window_oracle(self): + """NUM_STEPS > 0 with several token blocks per row (the long-context + launch the flashinfer draft backend uses) must reproduce the windowed + read plan: the closed-form kv_indptr, the [sink + recent] + tree slots, + and bit-for-bit the historical one-block grid, for every block count. + """ + for seq_lens, num_steps, topk, page_size, window, sink in [ + ([5000, 900], 4, 1, 1, 1024, 64), + ([5000, 900], 4, 2, 1, 1024, 0), # pure recent window, tree + ([300, 130], 3, 2, 1, 4096, 64), # cap exceeds seq_len -> keep all + ([600, 250], 3, 4, 16, 128, 16), # paged tree extend + ([262145, 131072], 4, 1, 1, 4032, 64), # long context + ([65536, 1029306], 3, 2, 1, 4032, 64), # long-context tree + ([1029306], 4, 1, 1, 0, 0), # window off, one long row + ]: + io = _make_inputs(seq_lens, num_steps, topk, page_size=page_size) + ref_indices, ref_indptr = _run(io, window_size=window, sink_size=sink) + for nb in (2, 3, 33): + with self.subTest( + seq_lens=seq_lens, topk=topk, window=window, sink=sink, nb=nb + ): + self._assert_slot_layout(io, window, sink, num_token_blocks=nb) + out_indices, out_indptr = _run( + io, window_size=window, sink_size=sink, num_token_blocks=nb + ) + self.assertTrue(torch.equal(out_indptr, ref_indptr)) + self.assertTrue(torch.equal(out_indices, ref_indices)) + def test_tree_topk_paged(self): """page_size > 1 AND topk > 1: the paged-tree extend branch. diff --git a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py index 5c2f5ee736..c889f362c9 100644 --- a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py +++ b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py @@ -84,17 +84,28 @@ class TestSpecKvIndicesGrid(CustomTestCase): (4, torch.int64), ]: inputs = _draft_inputs(seqs, topk, steps, "cuda", idx_dtype) - ref = _run_draft(inputs, topk, steps, page_size, 1, {}) - for nb in [ - 1, - kv_indices_num_token_blocks(POOL_LEN, steps * len(seqs) * topk), - triton.cdiv(POOL_LEN, 8192), - ]: - out = _run_draft( - inputs, topk, steps, page_size, nb, {"NUM_STEPS": steps} - ) - self.assertTrue(torch.equal(ref[0], out[0]), (seqs, topk, nb)) - self.assertTrue(torch.equal(ref[1], out[1]), (seqs, topk, nb)) + # window_size > 0: the StreamingLLM draft window, which the + # token-block path must honour exactly like the serial grid + for window_kw in ({}, {"window_size": 4032, "sink_size": 64}): + ref = _run_draft(inputs, topk, steps, page_size, 1, window_kw) + for nb in [ + 1, + kv_indices_num_token_blocks( + POOL_LEN, steps * len(seqs) * topk + ), + triton.cdiv(POOL_LEN, 8192), + ]: + out = _run_draft( + inputs, + topk, + steps, + page_size, + nb, + {"NUM_STEPS": steps, **window_kw}, + ) + key = (seqs, topk, nb, window_kw) + self.assertTrue(torch.equal(ref[0], out[0]), key) + self.assertTrue(torch.equal(ref[1], out[1]), key) def test_draft_reference(self): torch.manual_seed(1) diff --git a/test/registered/unit/mem_cache/test_kv_loc_plan.py b/test/registered/unit/mem_cache/test_kv_loc_plan.py index 341eb47f57..a32debace8 100644 --- a/test/registered/unit/mem_cache/test_kv_loc_plan.py +++ b/test/registered/unit/mem_cache/test_kv_loc_plan.py @@ -93,8 +93,11 @@ class _HostStreamGather: out, row_stride, ENTRY_PAGE_SIZE=1, + TOKEN_BLOCK_PARALLEL=False, token_mapping=None, ): + # TOKEN_BLOCK_PARALLEL only tiles the gather across programs; the + # host emulation writes each lane whole, so any value is correct. assert row_stride == ids.stride(0) for b in range(int(seq_lens.numel())): start = 0 if kv_start_idx is None else int(kv_start_idx[b])