[Perf] speculative: Parallelize EAGLE spec-decode KV index building over token blocks (flashinfer draft decode; verify/draft-extend for flashinfer, flashinfer_mla, aiter) (#42803)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: YAMY1234 <74099316+YAMY1234@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
YAMY1234
Po-Han Huang
Po-Han Huang
parent
4b9936b5e3
commit
991b846227
@@ -1,3 +1,5 @@
|
||||
from typing import Optional
|
||||
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
@@ -18,6 +20,41 @@ def kv_indices_num_token_blocks(table_width: int, base_programs: int) -> int:
|
||||
return max(1, min(cap, want))
|
||||
|
||||
|
||||
# Token-block parallelism pays only on long-context servers; below this the
|
||||
# spec-decode index launches keep their historical grids.
|
||||
SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768
|
||||
|
||||
|
||||
def spec_kv_index_token_blocks(
|
||||
table_width: int,
|
||||
kv_lens_sum: Optional[int],
|
||||
batch_size: int,
|
||||
base_programs: Optional[int] = None,
|
||||
length_cap: Optional[int] = None,
|
||||
context_len: Optional[int] = None,
|
||||
) -> int:
|
||||
"""Token blocks per base program for a spec-decode KV-index launch.
|
||||
|
||||
1 below the long-context gate (``context_len``, default ``table_width``).
|
||||
Otherwise sized from an upper bound on the longest live row — the launch
|
||||
time is set by the longest row, and ``kv_lens_sum`` bounds it from above
|
||||
while still a host int (tightened by ``length_cap`` and ``table_width``,
|
||||
which also bound any single row). Any block count is correct; it only
|
||||
changes the parallelism.
|
||||
"""
|
||||
gate = table_width if context_len is None else context_len
|
||||
if gate < SPEC_KV_INDEX_BLOCKS_MIN_CONTEXT or batch_size <= 0:
|
||||
return 1
|
||||
if kv_lens_sum is None:
|
||||
width = table_width
|
||||
else:
|
||||
width = min(kv_lens_sum, table_width)
|
||||
if length_cap is not None:
|
||||
width = min(width, length_cap)
|
||||
programs = batch_size if base_programs is None else base_programs
|
||||
return kv_indices_num_token_blocks(width, programs)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def create_flashinfer_kv_indices_triton(
|
||||
req_to_token_ptr, # [max_batch, max_context_len] token table; at
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.kernels.ops.attention.kv_indices import (
|
||||
create_flashinfer_kv_indices_triton,
|
||||
create_flashmla_kv_indices_triton,
|
||||
get_num_kv_index_blocks_flashmla,
|
||||
kv_indices_num_token_blocks,
|
||||
spec_kv_index_token_blocks,
|
||||
)
|
||||
from sglang.kernels.ops.attention.utils import assert_buffer_fits
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
@@ -155,13 +155,6 @@ fast_mode = False
|
||||
intra_batch_mode = True if _use_mla_ps_kernel else False
|
||||
|
||||
|
||||
# Token-block parallel KV-index building is enabled only where it pays:
|
||||
# the speculative-decoding paths (target_verify / draft_extend / draft
|
||||
# decode) of long-context servers. Everything else keeps the historical
|
||||
# one-program-per-request launch.
|
||||
_KV_INDEX_BLOCKS_MIN_CONTEXT = 32768
|
||||
|
||||
|
||||
class WrapperDispatch(Enum):
|
||||
SLIDING_WINDOW = auto()
|
||||
CROSS_ATTENTION = auto()
|
||||
@@ -1968,9 +1961,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
return output, final_lse
|
||||
|
||||
def _kv_index_blocks(self, bs: int) -> int:
|
||||
if self.max_context_len < _KV_INDEX_BLOCKS_MIN_CONTEXT:
|
||||
return 1
|
||||
return kv_indices_num_token_blocks(self.req_to_token.shape[1], bs)
|
||||
return spec_kv_index_token_blocks(
|
||||
self.req_to_token.shape[1],
|
||||
kv_lens_sum=None,
|
||||
batch_size=bs,
|
||||
context_len=self.max_context_len,
|
||||
)
|
||||
|
||||
def init_forward_metadata_out_graph(
|
||||
self,
|
||||
@@ -5125,12 +5121,12 @@ class AiterMultiStepDraftBackend:
|
||||
bs = self.topk * num_seqs
|
||||
seq_lens_sum = forward_batch.seq_lens_sum
|
||||
|
||||
num_token_blocks = (
|
||||
kv_indices_num_token_blocks(
|
||||
self.pool_len, self.speculative_num_steps * num_seqs * self.topk
|
||||
)
|
||||
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
|
||||
else 1
|
||||
num_token_blocks = spec_kv_index_token_blocks(
|
||||
self.pool_len,
|
||||
kv_lens_sum=None,
|
||||
batch_size=num_seqs,
|
||||
base_programs=self.speculative_num_steps * num_seqs * self.topk,
|
||||
context_len=self.max_context_len,
|
||||
)
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
|
||||
@@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Callable, List, Optional, Union
|
||||
import torch
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.kernels.ops.attention.kv_indices import spec_kv_index_token_blocks
|
||||
from sglang.kernels.ops.attention.utils import (
|
||||
assert_buffer_fits,
|
||||
)
|
||||
@@ -67,7 +68,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
if envs.SGLANG_ENABLE_TORCH_COMPILE.get():
|
||||
torch._logging.set_logs(dynamo=logging.ERROR)
|
||||
torch._dynamo.config.suppress_errors = True
|
||||
@@ -2512,6 +2512,21 @@ class FlashInferMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
# Blocks from the length sum (upper bound on the longest live row),
|
||||
# capped by the draft window, not from the table width. pool_len is
|
||||
# the req_to_token row width, the same table width the other call
|
||||
# sites size from.
|
||||
num_token_blocks = spec_kv_index_token_blocks(
|
||||
table_width=self.pool_len,
|
||||
kv_lens_sum=seq_lens_sum,
|
||||
batch_size=num_seqs,
|
||||
base_programs=self.speculative_num_steps * num_seqs * self.topk,
|
||||
length_cap=(
|
||||
self.draft_window_size + self.draft_sink_size
|
||||
if self.draft_window_size > 0
|
||||
else None
|
||||
),
|
||||
)
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
@@ -2532,8 +2547,11 @@ class FlashInferMultiStepDraftBackend:
|
||||
if use_host_indptr
|
||||
else 1
|
||||
)
|
||||
# One launch, the larger of the two tilings: the host-plan path never
|
||||
# drops below its 512-token blocks.
|
||||
num_token_blocks = max(num_token_blocks, num_blocks)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps * num_blocks, num_seqs, self.topk)
|
||||
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
|
||||
](
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
@@ -2553,7 +2571,14 @@ class FlashInferMultiStepDraftBackend:
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0,
|
||||
# A single token block without the host plan is the historical
|
||||
# launch; NUM_STEPS=0 keeps its 128-wide program instead of the
|
||||
# token-block specialization.
|
||||
NUM_STEPS=(
|
||||
self.speculative_num_steps
|
||||
if num_token_blocks > 1 or use_host_indptr
|
||||
else 0
|
||||
),
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -69,6 +69,7 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.attention.kv_indices import (
|
||||
create_flashinfer_kv_indices_triton,
|
||||
spec_kv_index_token_blocks,
|
||||
)
|
||||
from sglang.kernels.ops.kvcache.kv_read_table import (
|
||||
build_kv_read_table,
|
||||
@@ -538,14 +539,28 @@ class KVIndexTranslator:
|
||||
kv_start_idx: Optional[torch.Tensor] = None,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
token_mapping: Optional[torch.Tensor] = None,
|
||||
num_token_blocks: Optional[int] = 1,
|
||||
kv_lens_sum: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
|
||||
over; return whether they are physical. Packed from the plan's table
|
||||
once a table reader has had it built, else gathered and translated
|
||||
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
|
||||
(a static pool, or DCP) for the caller to finish; a static SWA pool can
|
||||
fuse its full->swa table into the gather as ``token_mapping``."""
|
||||
fuse its full->swa table into the gather as ``token_mapping``.
|
||||
``num_token_blocks`` tiles each request's gather over that many
|
||||
programs (any count is correct); ``None`` sizes it here with
|
||||
``spec_kv_index_token_blocks`` from ``kv_lens_sum``, the batch's
|
||||
host-int length sum (an upper bound on the longest row; ``None``
|
||||
falls back to the table width). The packed-table build path has its
|
||||
own parallelism and ignores the count."""
|
||||
bs = int(seq_lens.numel())
|
||||
if num_token_blocks is None:
|
||||
num_token_blocks = spec_kv_index_token_blocks(
|
||||
table_width=self.req_to_token.size(1),
|
||||
kv_lens_sum=kv_lens_sum,
|
||||
batch_size=bs,
|
||||
)
|
||||
if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
@@ -576,7 +591,7 @@ class KVIndexTranslator:
|
||||
)
|
||||
assert src.v2p is None, "pack_read_stream: a table-less translating read"
|
||||
assert token_mapping is None or not src.is_translated
|
||||
create_flashinfer_kv_indices_triton[(bs,)](
|
||||
create_flashinfer_kv_indices_triton[(bs, num_token_blocks)](
|
||||
src.ids,
|
||||
src.row_ids[:bs],
|
||||
seq_lens,
|
||||
@@ -585,6 +600,7 @@ class KVIndexTranslator:
|
||||
out,
|
||||
src.row_stride,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
TOKEN_BLOCK_PARALLEL=num_token_blocks > 1,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
return src.is_translated or token_mapping is not None
|
||||
|
||||
@@ -125,6 +125,11 @@ class EagleVerifyInput(SpecInput):
|
||||
indptr=cum_kv_seq_len,
|
||||
out=kv_indices,
|
||||
kind=kind,
|
||||
# The translator tiles the gather over token blocks, sized from
|
||||
# the length sum (an upper bound on the longest live row) while
|
||||
# it is still a host int.
|
||||
num_token_blocks=None,
|
||||
kv_lens_sum=paged_kernel_lens_sum,
|
||||
)
|
||||
mask_numel = (
|
||||
paged_kernel_lens_sum * self.draft_token_num
|
||||
@@ -429,6 +434,10 @@ class EagleDraftExtendInput(SpecInput):
|
||||
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=device)
|
||||
cum_kv_seq_len[1:] = torch.cumsum(paged_kernel_lens, dim=0)
|
||||
|
||||
# Captured while the length sum is still a host int (None -> the
|
||||
# translator sizes from its table width).
|
||||
host_kv_lens_sum = paged_kernel_lens_sum
|
||||
|
||||
if paged_kernel_lens_sum is None:
|
||||
paged_kernel_lens_sum = cum_kv_seq_len[-1]
|
||||
|
||||
@@ -443,5 +452,7 @@ class EagleDraftExtendInput(SpecInput):
|
||||
indptr=cum_kv_seq_len,
|
||||
out=kv_indices,
|
||||
kind=kind,
|
||||
num_token_blocks=None,
|
||||
kv_lens_sum=host_kv_lens_sum,
|
||||
)
|
||||
return kv_indices, cum_kv_seq_len, qo_indptr, None
|
||||
|
||||
Reference in New Issue
Block a user