[AMD][DCP 3/N] add aiter asm for kimi k3 target verify (#34537)

This commit is contained in:
billishyahao
2026-10-08 09:13:03 -07:00
committed by GitHub
parent 3030e31ff8
commit 4ab4d90d78
4 changed files with 619 additions and 65 deletions
@@ -125,6 +125,45 @@ def create_mla_kv_page_table_for_dcp(
)
@triton.jit
def compact_dcp_verify_token_table_to_ragged(
token_table_ptr, # in: [bs * q_len, table_stride] from create_mla_kv_page_table_for_dcp
local_kv_lens_ptr, # in: [bs] this rank's shard length per request, contiguous
kv_indptr_ptr, # in: [bs + 1] cumsum of the lengths, each clamped to >= 1
kv_indices_ptr, # out: ragged ids, one run per request
table_stride: tl.constexpr,
Q_LEN: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Rectangular token table -> ragged (kv_indices, kv_indptr).
Every row of a request repeats that request's shard, so this reads row 0.
"""
# The grid is (bs,) and the token loop is inside the kernel, so a cuda
# graph can capture this launch.
req = tl.program_id(0)
out_offset = tl.load(kv_indptr_ptr + req)
local_len = tl.load(local_kv_lens_ptr + req).to(tl.int32)
# The table must be wide enough for the whole shard. A narrow table makes
# this kernel read the next query row and return the wrong ids.
tl.device_assert(local_len <= table_stride)
if local_len == 0:
# Give an empty shard one slot, because the attention kernel fails
# on a shard of length zero. The caller removes that row later.
tl.store(kv_indices_ptr + out_offset, 0)
else:
row = token_table_ptr + req * Q_LEN * table_stride
for i in range(tl.cdiv(local_len, BLOCK_SIZE)):
offset = tl.arange(0, BLOCK_SIZE).to(tl.int64) + i * BLOCK_SIZE
mask = offset < local_len
tl.store(
kv_indices_ptr + out_offset + offset,
tl.load(row + offset, mask=mask),
mask=mask,
)
@triton.jit
def create_dcp_kv_indices(
kv_indptr,
+12 -4
View File
@@ -193,6 +193,10 @@ class EnvIntWithAlias(_DeprecatedEnvFallback, EnvInt):
pass
class EnvStrWithAlias(_DeprecatedEnvFallback, EnvStr):
pass
class EnvFloat(EnvField):
def parse(self, value: str) -> float:
try:
@@ -964,10 +968,14 @@ class Envs:
# import and Triton cga_layout prerequisites hold. Set to 0 to force the
# zero-pad mla_decode_fwd fallback (benchmarking / emergency disable).
SGLANG_AITER_MLA_GLUON = EnvBool(True)
# Select the AITER MLA kernel for DCP decode. "gluon" preserves the native
# low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and
# runs mla_decode_fwd with persistent metadata and return_lse=True.
SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon")
# Select the AITER MLA kernel for decode, "asm" or "gluon".
SGLANG_AITER_MLA_DECODE_BACKEND = EnvStrWithAlias(
"asm", deprecated_name="SGLANG_AITER_MLA_DCP_DECODE_BACKEND"
)
# Select the AITER MLA kernel for target verify, "asm" or "gluon".
SGLANG_AITER_MLA_VERIFY_BACKEND = EnvStr("asm")
# Let aiter plan the KV splits for the asm persistent MLA decode.
SGLANG_AITER_MLA_AUTO_KV_SPLITS = EnvBool(False)
# DSV4 Aiter flags
SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False)
@@ -72,7 +72,10 @@ except ImportError:
"aiter is AMD specific kernel library. Please make sure aiter is installed on your AMD device."
)
from sglang.kernels.ops.attention.dcp_kernels import create_mla_kv_page_table_for_dcp
from sglang.kernels.ops.attention.dcp_kernels import (
compact_dcp_verify_token_table_to_ragged,
create_mla_kv_page_table_for_dcp,
)
from sglang.kernels.ops.attention.merge_state import merge_state_triton
from sglang.kernels.ops.attention.utils import (
launch_reshape_and_cache_flash,
@@ -104,6 +107,37 @@ _use_fp8_prefill_attn = (
get_bool_env_var("SGLANG_AITER_FP8_PREFILL_ATTN", "True") and is_gfx95_supported()
)
# Max qlen limitation of the asm ps kernels to handle q with bf16 dtype
_MLA_ASM_PS_MAX_QLEN = 4
# (query head count -> query lengths) at which aiter's scheduler treats the
# count as native, on gfx950 with an fp8 query and KV. None means any length.
# This map is copied from natively_supported in aiter/ops/attention.py. See
# https://github.com/ROCm/aiter/blob/bf37db00749722ae29ac27c101f534da43c586f5/aiter/ops/attention.py#L1669-L1706
_MLA_ASM_CPRR_NATIVE_QLENS = {
16: None,
32: frozenset({4}),
96: frozenset(range(1, 6 + 1)),
128: None,
}
def _asm_cprr_supports_shape(heads: int, q_len: int) -> bool:
"""Whether aiter schedules this head count natively at this query length."""
if heads not in _MLA_ASM_CPRR_NATIVE_QLENS:
return False
allowed = _MLA_ASM_CPRR_NATIVE_QLENS[heads]
return allowed is None or q_len in allowed
def _asm_cprr_kernel_heads(gathered_heads: int, q_len: int) -> int:
"""The smallest supported head count at or above the gathered one, or 0."""
for heads in sorted(_MLA_ASM_CPRR_NATIVE_QLENS):
if heads >= gathered_heads and _asm_cprr_supports_shape(heads, q_len):
return heads
return 0
# (v_head_dim -> query head counts) that aiter's mla_reduce_v1 has an
# instantiation. This map is copied from MLA_REDUCE_ROUTER in
# aiter/csrc/kernels/mla/reduce.cu. See https://github.com/ROCm/aiter/blob/7915f53a4225b3f9cb632a97a23369dbebcf1be0/csrc/kernels/mla/reduce.cu#L923
@@ -186,6 +220,13 @@ class ForwardMetadata:
prefill_ps_metadata: Optional[MlaPrefillPsMetadata] = None
chunked_skip_prefix_ps_metadata: Optional[MlaPrefillPsMetadata] = None
chunked_prefix_ps_metadatas: Optional[list[Optional[MlaPrefillPsMetadata]]] = None
# cprr
global_kv_indptr: Optional[torch.Tensor] = None
use_asm_cprr_verify: bool = False
cprr_kernel_heads: int = 0
cprr_kv_indices: Optional[torch.Tensor] = None
cprr_kv_indptr: Optional[torch.Tensor] = None
cprr_empty_rows: Optional[torch.Tensor] = None
def _build_paged_kv_view(
@@ -313,6 +354,9 @@ class AiterAttnBackend(AttentionBackend):
# can never carry more seqs than the pool.
extend_dummy_seqs_capped_by_req_pool: bool = True
# declare here to avoid CI failure in test_aiter_fp8_q_unified_attention.py
use_mla_auto_kv_splits: bool = False
def __init__(
self,
model_runner: ModelRunner,
@@ -350,18 +394,33 @@ class AiterAttnBackend(AttentionBackend):
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
self.dcp_world_size = get_parallel().attn_dcp_size
self.mla_dcp_decode_backend = (
envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower()
)
if self.mla_dcp_decode_backend not in ("gluon", "asm"):
self.mla_decode_backend = envs.SGLANG_AITER_MLA_DECODE_BACKEND.get().lower()
if self.mla_decode_backend not in ("gluon", "asm"):
raise ValueError(
"SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', "
f"got {self.mla_dcp_decode_backend!r}"
"SGLANG_AITER_MLA_DECODE_BACKEND must be 'gluon' or 'asm', "
f"got {self.mla_decode_backend!r}"
)
self.use_mla_dcp_asm = (
self.use_mla
and self.dcp_world_size > 1
and self.mla_dcp_decode_backend == "asm"
and self.mla_decode_backend == "asm"
)
self.mla_verify_backend = envs.SGLANG_AITER_MLA_VERIFY_BACKEND.get().lower()
if self.mla_verify_backend not in ("gluon", "asm"):
raise ValueError(
"SGLANG_AITER_MLA_VERIFY_BACKEND must be 'gluon' or 'asm', "
f"got {self.mla_verify_backend!r}"
)
self._cprr_launch_logged = False
# fast_mode / intra_batch_mode for the cp verify schedule. Not the
# module defaults: those size reduce_partial_map at 65536 entries
# instead of 1276, and mla_decode_fwd then asks for a 64 GiB logits
# buffer.
self.cprr_fast_mode = True
self.cprr_intra_batch_mode = False
gathered_heads = self.num_head * self.dcp_world_size
self.cprr_kernel_heads = _asm_cprr_kernel_heads(
gathered_heads, self.num_draft_tokens or 1
)
# Get v_head_dim based on model type
@@ -570,10 +629,10 @@ class AiterAttnBackend(AttentionBackend):
)
global _use_mla_ps_kernel, fast_mode, intra_batch_mode
# current mla_decode_fwd only support fake-nps in num_head == 16
# so all num_head size does not use qh16 kernel to simulate
# it should not use fake-nps (fast_mode = False, intra_batch_mode = True)
# it will cause gpu-fault or accuracy issue.
# fake-nps (fast_mode False, intra_batch_mode True) picks aiter's
# v1_0 scheduler, which folds these counts to 16 while the gfx950
# fp8 launch does not. Both sides must fold alike or the kernel
# faults. At 16 neither folds, so fake-nps is correct there.
if self.mla_kernel_num_head_padded in (32, 64, 128):
fast_mode = True
intra_batch_mode = False
@@ -592,14 +651,16 @@ class AiterAttnBackend(AttentionBackend):
_use_mla_ps_kernel = False
fast_mode = False
intra_batch_mode = False
# Zero-pad topology (h12->qh16): prefer Gluon decode over PS kernel.
if self.head_pad_mode == "zero" and self.kv_cache_dtype == fp8_dtype:
# Disable ps only when gluon kernel is selected to avoid falling
# back to incorrect aiter kernel
if prefer_mla_gluon_decode(
head_pad_mode=self.head_pad_mode,
num_head=self.num_head,
kv_cache_dtype=self.kv_cache_dtype,
if (
prefer_mla_gluon_decode(
head_pad_mode=self.head_pad_mode,
num_head=self.num_head,
kv_cache_dtype=self.kv_cache_dtype,
)
and not self._asm_ps_supports_decode_and_verify()
):
_use_mla_ps_kernel = False
fast_mode = False
@@ -612,6 +673,27 @@ class AiterAttnBackend(AttentionBackend):
if self.num_draft_tokens is None and use_any_mla_persist:
self.max_split_per_batch = 64
# When the env var is on, aiter plans the KV splits. The default
# scheduler gives every request ceil(num_cu / batch_size) splits.
# It ignores the KV length. That clamp is inside the kernel, so no
# host value lifts it. The v1_2 scheduler sizes the count from the
# workload. A max_split_per_batch of -1 selects its auto branch.
#
# Under DCP the cp verify route already runs the v1_2 scheduler:
# make_mla_decode_meta_data_buffer takes cprr_fast_mode and
# cprr_intra_batch_mode instead. There only the -1 has an effect.
self.use_mla_auto_kv_splits = (
use_any_mla_persist and envs.SGLANG_AITER_MLA_AUTO_KV_SPLITS.get()
)
if self.use_mla_auto_kv_splits:
logger.info(
"aiter MLA: aiter plans the KV splits "
"(SGLANG_AITER_MLA_AUTO_KV_SPLITS=1)"
)
fast_mode = True
intra_batch_mode = False
self.max_split_per_batch = -1
self.fix_max_split_per_batch = self.max_split_per_batch
def pad_heads(self, x: torch.Tensor, padded: int) -> torch.Tensor:
@@ -688,13 +770,21 @@ class AiterAttnBackend(AttentionBackend):
*,
metadata_fast_mode: Optional[bool] = None,
metadata_intra_batch_mode: Optional[bool] = None,
nhead_override: Optional[int] = None,
max_split_per_batch: Optional[int] = None,
):
"""Allocate the PS work-schedule buffers, sized for nhead_override."""
# Under DCP this is the gathered head count (num_head * dcp_world_size);
# equals num_head_padded when DCP is off.
nhead = self.mla_kernel_num_head_padded
nhead = (
self.mla_kernel_num_head_padded
if nhead_override is None
else nhead_override
)
dtype = self.kv_cache_dtype
if self.attn_dp_enabled:
# The auto count is -1, which this min() would turn into a real cap.
if self.attn_dp_enabled and not self.use_mla_auto_kv_splits:
gpu = torch.cuda.current_device()
device_properties = torch.cuda.get_device_properties(gpu)
cu_num = device_properties.multi_processor_count
@@ -728,6 +818,11 @@ class AiterAttnBackend(AttentionBackend):
fast_mode=metadata_fast_mode,
num_kv_splits=self.max_split_per_batch,
intra_batch_mode=metadata_intra_batch_mode,
**(
{}
if max_split_per_batch is None
else {"max_split_per_batch": max_split_per_batch}
),
)
# aiter implementation
@@ -777,16 +872,23 @@ class AiterAttnBackend(AttentionBackend):
fast_mode,
max_split_per_batch,
intra_batch_mode,
is_cp_round_robin=False,
nhead_override=None,
):
nhead_kv = 1
page_size = self.page_size
dtype = self.kv_cache_dtype
nhead = (
self.mla_kernel_num_head_padded
if nhead_override is None
else nhead_override
)
meta = get_mla_metadata_v1(
qo_indptr,
kv_indptr,
kv_last_page_len,
self.mla_kernel_num_head_padded // nhead_kv,
nhead // nhead_kv,
nhead_kv,
False,
work_metadata,
@@ -801,6 +903,7 @@ class AiterAttnBackend(AttentionBackend):
fast_mode=fast_mode,
max_split_per_batch=max_split_per_batch,
intra_batch_mode=intra_batch_mode,
is_cp_round_robin=is_cp_round_robin,
dtype_q=dtype,
dtype_kv=dtype,
)
@@ -1287,6 +1390,52 @@ class AiterAttnBackend(AttentionBackend):
f"Got topk={self.topk}."
)
def _asm_ps_supports_decode_and_verify(self) -> bool:
"""Whether the asm PS MLA kernels support both decode and target verify."""
verify_q_len = self.num_draft_tokens or 1
if self.dcp_world_size > 1:
if _asm_cprr_supports_shape(self.cprr_kernel_heads, verify_q_len):
return True
logger.info(
"aiter MLA decode and verify stay on the Gluon kernel: the cp "
"verify kernel supports no %d-token window at %d gathered heads.",
verify_q_len,
self.num_head * self.dcp_world_size,
)
return False
if self._asm_ps_supports_qlen(verify_q_len):
return True
logger.info(
"aiter MLA decode and verify stay on the Gluon kernel: target "
"verify is %d tokens wide and the asm kernels support at most %d at "
"%d heads. A DSPARK block size of %d or lower would fit.",
verify_q_len,
_MLA_ASM_PS_MAX_QLEN,
self.mla_kernel_num_head_padded,
_MLA_ASM_PS_MAX_QLEN - 1,
)
return False
def _asm_ps_supports_qlen(self, q_len: int) -> bool:
"""Whether the asm PS dispatch has a kernel for this query length."""
return q_len <= _MLA_ASM_PS_MAX_QLEN or (
self.kv_cache_dtype == fp8_dtype
and self.mla_kernel_num_head_padded == 16
and is_gfx95_supported()
)
def _asm_cprr_supports_verify_shape(self, q_len: Optional[int]) -> bool:
"""Whether this topology has an asm cp kernel for DCP target verify."""
return (
_use_mla_ps_kernel
and self.mla_verify_backend == "asm"
and self.dcp_world_size > 1
and self.kv_cache_dtype == fp8_dtype
and q_len is not None
and self.cprr_kernel_heads > 0
and _asm_cprr_supports_shape(self.cprr_kernel_heads, q_len)
)
def _mla_decode_fwd_with_head_pad(
self,
q: torch.Tensor,
@@ -1301,6 +1450,27 @@ class AiterAttnBackend(AttentionBackend):
q / o must already be shaped (..., num_head, head_dim).
"""
num_head = layer.tp_q_head_num
if (
_use_mla_ps_kernel
and self.dcp_world_size <= 1
and self._asm_ps_supports_qlen(kwargs.get("max_seqlen_q") or 1)
):
# The asm kernel reads KV row 0 for masked positions and weights it
# by p = 0. A NaN there spreads to every request, because
# 0 * NaN = NaN. Row 0 is where CUDA-graph padding tokens write
# their KV, and that KV can be NaN. Gluon, the kernel this path
# replaces, does not read row 0.
k_buffer_flat[0].zero_()
if (
self.kv_cache_dtype == fp8_dtype
and self.mla_kernel_num_head_padded == 16
and is_gfx95_supported()
and (kwargs.get("max_seqlen_q") or 1) > _MLA_ASM_PS_MAX_QLEN
and q.dtype != fp8_dtype
):
q_flat = q.reshape(q.shape[0], -1).to(fp8_dtype)
kwargs["q_scale"] = torch.ones((), dtype=torch.float32, device=q.device)
q = q_flat.view(q.shape)
if self.head_pad_mode == "repeat" or (
self.head_pad_mode == "none" and self.num_head_padded != self.num_head
):
@@ -1367,6 +1537,17 @@ class AiterAttnBackend(AttentionBackend):
return 1
return int(seq_lens.max().item())
def _kernel_num_kv_splits(self) -> Optional[int]:
"""The split count for a decode kernel call, None to let aiter pick.
The metadata call takes an int and gets -1 for the auto branch. The
kernel takes None instead. mla.py re-decides persistent mode only on
None. It would read -1 as a real count.
"""
if self.use_mla_auto_kv_splits:
return None
return self.forward_metadata.num_kv_splits
def _forward_mla_decode(
self,
q: torch.Tensor,
@@ -1382,6 +1563,10 @@ class AiterAttnBackend(AttentionBackend):
head_pad_mode=getattr(self, "head_pad_mode", "none"),
num_head=getattr(self, "num_head", layer.tp_q_head_num),
kv_cache_dtype=self.kv_cache_dtype,
) and not (
_use_mla_ps_kernel
and self.mla_decode_backend == "asm"
and self._asm_ps_supports_qlen(max_q_len)
):
return mla_gluon_decode(
q=q,
@@ -1401,7 +1586,7 @@ class AiterAttnBackend(AttentionBackend):
reduce_indptr = self.forward_metadata.reduce_indptr
reduce_final_map = self.forward_metadata.reduce_final_map
reduce_partial_map = self.forward_metadata.reduce_partial_map
num_kv_splits = self.forward_metadata.num_kv_splits
num_kv_splits = self._kernel_num_kv_splits()
return self._mla_decode_fwd_with_head_pad(
q,
@@ -1426,10 +1611,17 @@ class AiterAttnBackend(AttentionBackend):
num_kv_splits=num_kv_splits,
)
def _get_dcp_graph_max_local_kv_len(self) -> int:
"""Static upper bound on this rank's shard, ceil(max_context_len / W)."""
def _dcp_max_local_kv_len(self, max_global_kv_len: int) -> int:
"""Widest shard one rank can hold for a global kv length."""
w = max(self.dcp_world_size, 1)
return (self.max_context_len + w - 1) // w
return (max_global_kv_len + w - 1) // w
def _dcp_graph_max_local_kv_len(self) -> int:
"""Static upper bound on this rank's shard under a cuda graph."""
max_global_kv_len = self.max_context_len
if self._asm_cprr_supports_verify_shape(self.num_draft_tokens):
max_global_kv_len += self.num_draft_tokens
return self._dcp_max_local_kv_len(max_global_kv_len)
def _forward_decode_dcp(self, q, k_buffer, layer, k_descale):
"""Attend this rank's KV shard for decode -> (out, natural-log lse)."""
@@ -1446,8 +1638,12 @@ class AiterAttnBackend(AttentionBackend):
q_mla = q.view(bs, num_heads, layer.qk_head_dim)
q_scale = k_descale
if q_mla.dtype != fp8_dtype:
q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1))
q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim)
q_mla = (
q_mla.reshape(bs, -1)
.to(fp8_dtype)
.view(bs, num_heads, layer.qk_head_dim)
)
q_scale = torch.ones((), dtype=torch.float32, device=q.device)
out = torch.empty(
(bs, num_heads, layer.v_head_dim),
@@ -1465,7 +1661,7 @@ class AiterAttnBackend(AttentionBackend):
fm.max_q_len or 1,
sm_scale=layer.scaling,
logit_cap=layer.logit_cap,
num_kv_splits=fm.num_kv_splits,
num_kv_splits=self._kernel_num_kv_splits(),
work_meta_data=fm.work_metadata,
work_indptr=fm.work_indptr,
work_info_set=fm.work_info_set,
@@ -1496,13 +1692,126 @@ class AiterAttnBackend(AttentionBackend):
)
return out, lse.view(bs, num_heads)
def _forward_verify_dcp(self, q, k_window, layer, k_descale):
"""Attend the committed prefix and the verify window separately, then
merge -> (out, natural-log lse).
def _forward_verify_asm_cprr(self, q, layer, k_descale, n_rows):
"""DCP target verify on the asm cp round-robin kernel: one masked pass."""
fm = self.forward_metadata
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
kernel_heads = fm.cprr_kernel_heads or num_heads
if not self._cprr_launch_logged:
self._cprr_launch_logged = True
logger.info(
"aiter DCP cp verify: gathered=%d kernel_heads=%d%s "
"intra_batch=%s max_q_len=%d reduce_partial_map=%d",
num_heads,
kernel_heads,
" (padded)" if kernel_heads != num_heads else " (native)",
self.cprr_intra_batch_mode,
fm.max_q_len,
fm.reduce_partial_map.numel(),
)
# torch.full, not torch.as_tensor: as_tensor copies from host memory,
# which a cuda graph capture cannot record.
kv_scale = torch.full(
(),
self._resolve_fp8_kv_scale_float(layer, k_descale),
dtype=torch.float32,
device=q.device,
)
kv_indices, kv_indptr = fm.cprr_kv_indices, fm.cprr_kv_indptr
q_in = (
q.view(n_rows, -1).to(fp8_dtype).view(n_rows, num_heads, layer.qk_head_dim)
)
q_scale = torch.ones((), dtype=torch.float32, device=q.device)
if kernel_heads != num_heads:
q_in = self.pad_heads(q_in, kernel_heads)
out = q.new_empty(
(n_rows, kernel_heads, layer.v_head_dim), dtype=self.input_dtype
)
_, lse = mla_decode_fwd(
q_in,
self.token_to_kv_pool.get_key_buffer(layer.layer_id).view(
-1, 1, 1, layer.qk_head_dim
),
out,
fm.qo_indptr,
kv_indptr,
kv_indices,
fm.kv_last_page_len,
fm.max_q_len,
sm_scale=layer.scaling,
work_meta_data=fm.work_metadata,
work_indptr=fm.work_indptr,
work_info_set=fm.work_info_set,
reduce_indptr=fm.reduce_indptr,
reduce_final_map=fm.reduce_final_map,
reduce_partial_map=fm.reduce_partial_map,
q_scale=q_scale,
kv_scale=kv_scale,
intra_batch_mode=self.cprr_intra_batch_mode,
num_kv_splits=self._kernel_num_kv_splits(),
return_lse=True,
g_kv_indptr=fm.global_kv_indptr,
cp_world_size=self.dcp_world_size,
cp_rank=get_parallel().attn_dcp_rank,
causal=True,
)
lse = lse.view(n_rows, kernel_heads)
if kernel_heads != num_heads:
out = out[:, :num_heads, :]
lse = lse[:, :num_heads]
return out, lse
Splitting at the window boundary avoids the one thing the decode kernel
cannot do under DCP: mask on the GLOBAL position g(j) = j * W + r.
"""
def _build_dcp_verify_ragged_indices(
self,
verify_token_table: torch.Tensor,
local_kv_lens: torch.Tensor,
bs: int,
q_len: int,
out_indices: Optional[torch.Tensor] = None,
out_indptr: Optional[torch.Tensor] = None,
out_empty: Optional[torch.Tensor] = None,
):
"""This rank's verify shard as ragged (kv_indices, kv_indptr, empty_rows)."""
# Pass out_* to use the address-stable buffers of the cuda graph.
# Omit them to allocate new tensors.
# The .contiguous() call is necessary. The slice has stride q_len, and
# .to() returns an int32 tensor unchanged, so without the copy the
# kernel reads lens[req] instead of lens[req * q_len]. The two agree
# only at batch size 1.
lens = local_kv_lens[: bs * q_len : q_len].to(torch.int32).contiguous()
# The kernel faults on a zero-length shard, so give every shard a slot.
clamped = lens.clamp_min(1)
if out_indptr is None:
kv_indptr = torch.zeros(bs + 1, dtype=torch.int32, device=lens.device)
else:
kv_indptr = out_indptr[: bs + 1]
# Not kv_indptr[0] = 0: that copies from host memory, which a
# cuda graph capture cannot record.
kv_indptr[:1].zero_()
torch.cumsum(clamped, dim=0, out=kv_indptr[1:])
if out_indices is None:
kv_indices = verify_token_table.new_empty(
(int(clamped.sum().item()),), dtype=torch.int32
)
else:
kv_indices = out_indices
compact_dcp_verify_token_table_to_ragged[(bs,)](
verify_token_table,
lens,
kv_indptr,
kv_indices,
table_stride=verify_token_table.shape[1],
Q_LEN=q_len,
BLOCK_SIZE=_DCP_VERIFY_TABLE_COLS_PER_BLOCK,
)
empty_rows = local_kv_lens[: bs * q_len].eq(0)
if out_empty is not None:
empty_rows = out_empty[: bs * q_len].copy_(empty_rows)
return kv_indices, kv_indptr, empty_rows
def _forward_verify_dcp(self, q, k_window, layer, k_descale):
"""DCP target verify -> (out, natural-log lse) for the cross-rank merge."""
fm = self.forward_metadata
q_len = fm.max_q_len
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
@@ -1510,6 +1819,13 @@ class AiterAttnBackend(AttentionBackend):
n_rows = seqused_k.shape[0]
bs = n_rows // q_len
if fm.use_asm_cprr_verify:
out_a, lse_a = self._forward_verify_asm_cprr(q, layer, k_descale, n_rows)
empty = fm.cprr_empty_rows
out_a.masked_fill_(empty[:, None, None], 0.0)
lse_a.masked_fill_(empty[:, None], -1e30)
return out_a, lse_a
out_a, lse_a = mla_gluon_decode(
q=q.view(n_rows, num_heads, layer.qk_head_dim),
k_buffer=self.token_to_kv_pool.get_key_buffer(layer.layer_id),
@@ -1730,6 +2046,9 @@ class AiterAttnBackend(AttentionBackend):
# dcp metadata
local_kv_lens = None
verify_token_table = None
global_kv_indptr = None
use_asm_cprr_verify = False
cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None
if forward_batch.forward_mode.is_decode_or_idle():
if spec_info is None or forward_batch.forward_mode.is_idle():
kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0)
@@ -1967,7 +2286,8 @@ class AiterAttnBackend(AttentionBackend):
if self.use_mla:
draft_num = spec_info.draft_token_num
device = forward_batch.seq_lens.device
if self.dcp_world_size > 1:
use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(draft_num)
if self.dcp_world_size > 1 and not use_asm_cprr_verify:
kv_lens = forward_batch.seq_lens.to(torch.int32).clone()
kv_lens_sum = forward_batch.seq_lens_sum
else:
@@ -2001,6 +2321,7 @@ class AiterAttnBackend(AttentionBackend):
)
if self.dcp_world_size > 1:
global_kv_indptr = kv_indptr.clone()
self._plan_dcp_decode_metadata(
kv_indptr,
kv_indices,
@@ -2016,11 +2337,27 @@ class AiterAttnBackend(AttentionBackend):
forward_batch.req_pool_indices,
bs,
draft_num,
(max_kv_len + self.dcp_world_size - 1) // self.dcp_world_size,
self._dcp_max_local_kv_len(
max_kv_len + (draft_num if use_asm_cprr_verify else 0)
),
)
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
if use_asm_cprr_verify:
(
cprr_kv_indices,
cprr_kv_indptr,
cprr_empty_rows,
) = self._build_dcp_verify_ragged_indices(
verify_token_table, local_kv_lens, bs, draft_num
)
if _use_mla_ps_kernel and (
self.dcp_world_size <= 1 or use_asm_cprr_verify
):
max_seqlen_qo = draft_num
is_cp_round_robin = self.dcp_world_size > 1
metadata_heads = (
self.cprr_kernel_heads if use_asm_cprr_verify else None
)
(
work_metadata,
work_indptr,
@@ -2028,13 +2365,26 @@ class AiterAttnBackend(AttentionBackend):
reduce_indptr,
reduce_final_map,
reduce_partial_map,
) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, bs)
) = self.make_mla_decode_meta_data_buffer(
max_seqlen_qo,
bs,
metadata_fast_mode=(
self.cprr_fast_mode if use_asm_cprr_verify else None
),
metadata_intra_batch_mode=(
self.cprr_intra_batch_mode if use_asm_cprr_verify else None
),
nhead_override=metadata_heads,
max_split_per_batch=(
self.max_split_per_batch if use_asm_cprr_verify else None
),
)
num_kv_splits = self.max_split_per_batch
self.make_mla_meta_data(
qo_indptr,
kv_indptr,
cprr_kv_indptr if use_asm_cprr_verify else kv_indptr,
self.kv_last_page_len[:bs],
work_metadata,
work_info_set,
@@ -2043,9 +2393,17 @@ class AiterAttnBackend(AttentionBackend):
reduce_final_map,
reduce_partial_map,
max_seqlen_qo,
fast_mode=fast_mode,
fast_mode=(
self.cprr_fast_mode if use_asm_cprr_verify else fast_mode
),
max_split_per_batch=num_kv_splits,
intra_batch_mode=intra_batch_mode,
intra_batch_mode=(
self.cprr_intra_batch_mode
if use_asm_cprr_verify
else intra_batch_mode
),
is_cp_round_robin=is_cp_round_robin,
nhead_override=metadata_heads,
)
self.forward_metadata = ForwardMetadata(
@@ -2066,6 +2424,14 @@ class AiterAttnBackend(AttentionBackend):
run_graph=False,
local_kv_lens=local_kv_lens,
verify_token_table=verify_token_table,
global_kv_indptr=global_kv_indptr,
use_asm_cprr_verify=use_asm_cprr_verify,
cprr_kernel_heads=self.cprr_kernel_heads
if use_asm_cprr_verify
else 0,
cprr_kv_indices=cprr_kv_indices,
cprr_kv_indptr=cprr_kv_indptr,
cprr_empty_rows=cprr_empty_rows,
)
else:
draft_num = forward_batch.input_ids.shape[0] // bs
@@ -2395,10 +2761,8 @@ class AiterAttnBackend(AttentionBackend):
self.cuda_graph_verify_local_kv_lens = torch.zeros(
(n_verify_rows,), dtype=torch.int32, device=self.device
)
# Capture-stable token table, filled out-of-graph. One column per
# TOKEN, so the width is the worst-case shard ceil(ctx_len / W).
self.cuda_graph_verify_token_table = torch.zeros(
(n_verify_rows, self._get_dcp_graph_max_local_kv_len()),
(n_verify_rows, self._dcp_graph_max_local_kv_len()),
dtype=torch.int32,
device=self.device,
)
@@ -2406,10 +2770,24 @@ class AiterAttnBackend(AttentionBackend):
# verify plan without a sync; see _plan_dcp_decode_metadata.
self.cuda_graph_dcp_static_local_kv_lens = torch.full(
(max_bs,),
self._get_dcp_graph_max_local_kv_len(),
self._dcp_graph_max_local_kv_len(),
dtype=torch.int32,
device="cpu",
)
self.cuda_graph_global_kv_indptr = torch.zeros(
(max_bs + 1,), dtype=torch.int32, device=self.device
)
self.cuda_graph_cprr_kv_indptr = torch.zeros(
(max_bs + 1,), dtype=torch.int32, device=self.device
)
self.cuda_graph_cprr_kv_indices = torch.zeros(
(max_bs * (self._dcp_graph_max_local_kv_len() + 1),),
dtype=torch.int32,
device=self.device,
)
self.cuda_graph_cprr_empty_rows = torch.zeros(
(n_verify_rows,), dtype=torch.bool, device=self.device
)
if kv_indices_buf is None:
max_num_blocks_per_seq = (
self.max_context_len + self.page_size - 1
@@ -2471,6 +2849,9 @@ class AiterAttnBackend(AttentionBackend):
(True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode)
)
graph_use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(
self.num_draft_tokens
)
(
self.work_metadata,
self.work_indptr,
@@ -2481,8 +2862,37 @@ class AiterAttnBackend(AttentionBackend):
) = self.make_mla_decode_meta_data_buffer(
max_seqlen_qo,
max_bs,
metadata_fast_mode=metadata_fast_mode,
metadata_intra_batch_mode=metadata_intra_batch_mode,
metadata_fast_mode=(
self.cprr_fast_mode
if graph_use_asm_cprr_verify
else metadata_fast_mode
),
metadata_intra_batch_mode=(
self.cprr_intra_batch_mode
if graph_use_asm_cprr_verify
else metadata_intra_batch_mode
),
nhead_override=(
self.cprr_kernel_heads if graph_use_asm_cprr_verify else None
),
max_split_per_batch=(
self.max_split_per_batch if graph_use_asm_cprr_verify else None
),
)
logger.info(
"aiter DCP cp verify graph buffers: use_asm_cprr_verify=%s num_draft_tokens=%s "
"max_seqlen_qo=%d max_bs=%d nhead=%s intra=%s split_cap=%s "
"reduce_partial_map=%d",
graph_use_asm_cprr_verify,
self.num_draft_tokens,
max_seqlen_qo,
max_bs,
self.cprr_kernel_heads if graph_use_asm_cprr_verify else None,
self.cprr_intra_batch_mode
if graph_use_asm_cprr_verify
else metadata_intra_batch_mode,
self.max_split_per_batch if graph_use_asm_cprr_verify else None,
self.reduce_partial_map.numel(),
)
else:
@@ -2536,6 +2946,9 @@ class AiterAttnBackend(AttentionBackend):
# DCP metadata which will be populated for MLA decode when dcp enabled
local_kv_lens = None
verify_token_table = None
global_kv_indptr = None
cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None
use_asm_cprr_verify = False
swa_page_table = None
# Unified verify never reads this host max (replay sizes from
@@ -2712,9 +3125,12 @@ class AiterAttnBackend(AttentionBackend):
device=self.device,
)
if self.use_mla:
use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(
self.num_draft_tokens
)
kv_lens = (
seq_lens
if self.dcp_world_size > 1
if (self.dcp_world_size > 1 and not use_asm_cprr_verify)
else seq_lens + self.num_draft_tokens
)
else:
@@ -2748,6 +3164,8 @@ class AiterAttnBackend(AttentionBackend):
kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs]
if self.use_mla and self.dcp_world_size > 1:
global_kv_indptr = self.cuda_graph_global_kv_indptr[: bs + 1]
global_kv_indptr.copy_(kv_indptr[: bs + 1])
self._plan_dcp_decode_metadata(
kv_indptr,
kv_indices,
@@ -2767,19 +3185,36 @@ class AiterAttnBackend(AttentionBackend):
req_pool_indices,
bs,
self.num_draft_tokens,
self._get_dcp_graph_max_local_kv_len(),
self._dcp_graph_max_local_kv_len(),
out=self.cuda_graph_verify_token_table[:n_rows],
out_lens=self.cuda_graph_verify_local_kv_lens[:n_rows],
)
if use_asm_cprr_verify:
(
cprr_kv_indices,
cprr_kv_indptr,
cprr_empty_rows,
) = self._build_dcp_verify_ragged_indices(
verify_token_table,
local_kv_lens,
bs,
self.num_draft_tokens,
out_indices=self.cuda_graph_cprr_kv_indices,
out_indptr=self.cuda_graph_cprr_kv_indptr,
out_empty=self.cuda_graph_cprr_empty_rows,
)
if self.use_mla:
max_q_len = self.num_draft_tokens
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
if _use_mla_ps_kernel and (
self.dcp_world_size <= 1 or use_asm_cprr_verify
):
num_kv_splits = self.max_split_per_batch
is_cp_round_robin = self.dcp_world_size > 1
self.make_mla_meta_data(
qo_indptr,
kv_indptr,
cprr_kv_indptr if use_asm_cprr_verify else kv_indptr,
kv_last_page_len,
self.work_metadata,
self.work_info_set,
@@ -2788,9 +3223,19 @@ class AiterAttnBackend(AttentionBackend):
self.reduce_final_map,
self.reduce_partial_map,
max_q_len,
fast_mode=fast_mode,
fast_mode=(
self.cprr_fast_mode if use_asm_cprr_verify else fast_mode
),
max_split_per_batch=num_kv_splits,
intra_batch_mode=intra_batch_mode,
intra_batch_mode=(
self.cprr_intra_batch_mode
if use_asm_cprr_verify
else intra_batch_mode
),
is_cp_round_robin=is_cp_round_robin,
nhead_override=(
self.cprr_kernel_heads if use_asm_cprr_verify else None
),
)
work_metadata = self.work_metadata
@@ -2817,6 +3262,14 @@ class AiterAttnBackend(AttentionBackend):
num_kv_splits=num_kv_splits,
local_kv_lens=local_kv_lens,
verify_token_table=verify_token_table,
global_kv_indptr=global_kv_indptr,
use_asm_cprr_verify=use_asm_cprr_verify,
cprr_kernel_heads=self.cprr_kernel_heads
if use_asm_cprr_verify
else 0,
cprr_kv_indices=cprr_kv_indices,
cprr_kv_indptr=cprr_kv_indptr,
cprr_empty_rows=cprr_empty_rows,
)
else:
max_q_len = verify_tokens_per_req
@@ -3368,8 +3821,6 @@ class AiterAttnBackend(AttentionBackend):
forward_batch.forward_mode.is_target_verify()
and self.dcp_world_size > 1
):
# two-stage dcp verify, dispatched before the dims below: the
# model provides the per rank kvcache slices.
return self._forward_verify_dcp(q, k, layer, k_descale)
qk_nope_head_dim = k.shape[-1] - qk_rope_head_dim
@@ -3534,6 +3985,10 @@ class AiterAttnBackend(AttentionBackend):
head_pad_mode=getattr(self, "head_pad_mode", "none"),
num_head=getattr(self, "num_head", layer.tp_q_head_num),
kv_cache_dtype=self.kv_cache_dtype,
) and not (
_use_mla_ps_kernel
and self.mla_verify_backend == "asm"
and self._asm_ps_supports_qlen(self.forward_metadata.max_q_len or 1)
):
return mla_gluon_decode(
q=q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),