[Perf] speculative: Parallelize EAGLE spec-decode KV index building over token blocks (flashinfer draft decode; verify/draft-extend for flashinfer, flashinfer_mla, aiter) (#42803)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
Brandon Zhang
2026-10-21 15:26:35 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 YAMY1234 Po-Han Huang Po-Han Huang
parent 4b9936b5e3
commit 991b846227
8 changed files with 170 additions and 37 deletions
@@ -1,3 +1,5 @@
from typing import Optional
import triton
import triton.language as tl
@@ -18,6 +20,41 @@ def kv_indices_num_token_blocks(table_width: int, base_programs: int) -> int:
return max(1, min(cap, want))
# Token-block parallelism pays only on long-context servers; below this the
# spec-decode index launches keep their historical grids.
SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768
def spec_kv_index_token_blocks(
table_width: int,
kv_lens_sum: Optional[int],
batch_size: int,
base_programs: Optional[int] = None,
length_cap: Optional[int] = None,
context_len: Optional[int] = None,
) -> int:
"""Token blocks per base program for a spec-decode KV-index launch.
1 below the long-context gate (``context_len``, default ``table_width``).
Otherwise sized from an upper bound on the longest live row — the launch
time is set by the longest row, and ``kv_lens_sum`` bounds it from above
while still a host int (tightened by ``length_cap`` and ``table_width``,
which also bound any single row). Any block count is correct; it only
changes the parallelism.
"""
gate = table_width if context_len is None else context_len
if gate < SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT or batch_size <= 0:
return 1
if kv_lens_sum is None:
width = table_width
else:
width = min(kv_lens_sum, table_width)
if length_cap is not None:
width = min(width, length_cap)
programs = batch_size if base_programs is None else base_programs
return kv_indices_num_token_blocks(width, programs)
@triton.jit
def create_flashinfer_kv_indices_triton(
req_to_token_ptr, # [max_batch, max_context_len] token table; at
@@ -23,7 +23,7 @@ from sglang.kernels.ops.attention.kv_indices import (
create_flashinfer_kv_indices_triton,
create_flashmla_kv_indices_triton,
get_num_kv_index_blocks_flashmla,
kv_indices_num_token_blocks,
spec_kv_index_token_blocks,
)
from sglang.kernels.ops.attention.utils import assert_buffer_fits
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
@@ -155,13 +155,6 @@ fast_mode = False
intra_batch_mode = True if _use_mla_ps_kernel else False
# Token-block parallel KV-index building is enabled only where it pays:
# the speculative-decoding paths (target_verify / draft_extend / draft
# decode) of long-context servers. Everything else keeps the historical
# one-program-per-request launch.
_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768
class WrapperDispatch(Enum):
SLIDING_WINDOW = auto()
CROSS_ATTENTION = auto()
@@ -1968,9 +1961,12 @@ class AiterAttnBackend(AttentionBackend):
return output, final_lse
def _kv_index_blocks(self, bs: int) -> int:
if self.max_context_len < _KV_INDEX_BLOCKS_MIN_CONTEXT:
return 1
return kv_indices_num_token_blocks(self.req_to_token.shape[1], bs)
return spec_kv_index_token_blocks(
self.req_to_token.shape[1],
kv_lens_sum=None,
batch_size=bs,
context_len=self.max_context_len,
)
def init_forward_metadata_out_graph(
self,
@@ -5125,12 +5121,12 @@ class AiterMultiStepDraftBackend:
bs = self.topk * num_seqs
seq_lens_sum = forward_batch.seq_lens_sum
num_token_blocks = (
kv_indices_num_token_blocks(
self.pool_len, self.speculative_num_steps * num_seqs * self.topk
)
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
else 1
num_token_blocks = spec_kv_index_token_blocks(
self.pool_len,
kv_lens_sum=None,
batch_size=num_seqs,
base_programs=self.speculative_num_steps * num_seqs * self.topk,
context_len=self.max_context_len,
)
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Union
import torch
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.kernels.ops.attention.kv_indices import spec_kv_index_token_blocks
from sglang.kernels.ops.attention.utils import (
assert_buffer_fits,
)
@@ -67,7 +68,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
if envs.SGLANG_ENABLE_TORCH_COMPILE.get():
torch._logging.set_logs(dynamo=logging.ERROR)
torch._dynamo.config.suppress_errors = True
@@ -2512,6 +2512,21 @@ class FlashInferMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
# Blocks from the length sum (upper bound on the longest live row),
# capped by the draft window, not from the table width. pool_len is
# the req_to_token row width, the same table width the other call
# sites size from.
num_token_blocks = spec_kv_index_token_blocks(
table_width=self.pool_len,
kv_lens_sum=seq_lens_sum,
batch_size=num_seqs,
base_programs=self.speculative_num_steps * num_seqs * self.topk,
length_cap=(
self.draft_window_size + self.draft_sink_size
if self.draft_window_size > 0
else None
),
)
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
@@ -2532,8 +2547,11 @@ class FlashInferMultiStepDraftBackend:
if use_host_indptr
else 1
)
# One launch, the larger of the two tilings: the host-plan path never
# drops below its 512-token blocks.
num_token_blocks = max(num_token_blocks, num_blocks)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps * num_blocks, num_seqs, self.topk)
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
](
src.row_ids,
src.ids,
@@ -2553,7 +2571,14 @@ class FlashInferMultiStepDraftBackend:
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0,
# A single token block without the host plan is the historical
# launch; NUM_STEPS=0 keeps its 128-wide program instead of the
# token-block specialization.
NUM_STEPS=(
self.speculative_num_steps
if num_token_blocks > 1 or use_host_indptr
else 0
),
)
assert forward_batch.spec_info is not None
@@ -69,6 +69,7 @@ import torch
from sglang.kernels.ops.attention.kv_indices import (
create_flashinfer_kv_indices_triton,
spec_kv_index_token_blocks,
)
from sglang.kernels.ops.kvcache.kv_read_table import (
build_kv_read_table,
@@ -538,14 +539,28 @@ class KVIndexTranslator:
kv_start_idx: Optional[torch.Tensor] = None,
kind: IdSpaceKind = IdSpaceKind.FULL,
token_mapping: Optional[torch.Tensor] = None,
num_token_blocks: Optional[int] = 1,
kv_lens_sum: Optional[int] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
over; return whether they are physical. Packed from the plan's table
once a table reader has had it built, else gathered and translated
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
(a static pool, or DCP) for the caller to finish; a static SWA pool can
fuse its full->swa table into the gather as ``token_mapping``."""
fuse its full->swa table into the gather as ``token_mapping``.
``num_token_blocks`` tiles each request's gather over that many
programs (any count is correct); ``None`` sizes it here with
``spec_kv_index_token_blocks`` from ``kv_lens_sum``, the batch's
host-int length sum (an upper bound on the longest row; ``None``
falls back to the table width). The packed-table build path has its
own parallelism and ignores the count."""
bs = int(seq_lens.numel())
if num_token_blocks is None:
num_token_blocks = spec_kv_index_token_blocks(
table_width=self.req_to_token.size(1),
kv_lens_sum=kv_lens_sum,
batch_size=bs,
)
if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
@@ -576,7 +591,7 @@ class KVIndexTranslator:
)
assert src.v2p is None, "pack_read_stream: a table-less translating read"
assert token_mapping is None or not src.is_translated
create_flashinfer_kv_indices_triton[(bs,)](
create_flashinfer_kv_indices_triton[(bs, num_token_blocks)](
src.ids,
src.row_ids[:bs],
seq_lens,
@@ -585,6 +600,7 @@ class KVIndexTranslator:
out,
src.row_stride,
ENTRY_PAGE_SIZE=src.entry_page_size,
TOKEN_BLOCK_PARALLEL=num_token_blocks > 1,
token_mapping=token_mapping,
)
return src.is_translated or token_mapping is not None
@@ -125,6 +125,11 @@ class EagleVerifyInput(SpecInput):
indptr=cum_kv_seq_len,
out=kv_indices,
kind=kind,
# The translator tiles the gather over token blocks, sized from
# the length sum (an upper bound on the longest live row) while
# it is still a host int.
num_token_blocks=None,
kv_lens_sum=paged_kernel_lens_sum,
)
mask_numel = (
paged_kernel_lens_sum * self.draft_token_num
@@ -429,6 +434,10 @@ class EagleDraftExtendInput(SpecInput):
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=device)
cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0)
# Captured while the length sum is still a host int (None -> the
# translator sizes from its table width).
host_kv_lens_sum = paged_kernel_lens_sum
if paged_kernel_lens_sum is None:
paged_kernel_lens_sum = cum_kv_seq_len[-1]
@@ -443,5 +452,7 @@ class EagleDraftExtendInput(SpecInput):
indptr=cum_kv_seq_len,
out=kv_indices,
kind=kind,
num_token_blocks=None,
kv_lens_sum=host_kv_lens_sum,
)
return kv_indices, cum_kv_seq_len, qo_indptr, None