[AMD][DCP 3/N] add aiter asm for kimi k3 target verify (#34537)
This commit is contained in:
@@ -125,6 +125,45 @@ def create_mla_kv_page_table_for_dcp(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def compact_dcp_verify_token_table_to_ragged(
|
||||
token_table_ptr, # in: [bs * q_len, table_stride] from create_mla_kv_page_table_for_dcp
|
||||
local_kv_lens_ptr, # in: [bs] this rank's shard length per request, contiguous
|
||||
kv_indptr_ptr, # in: [bs + 1] cumsum of the lengths, each clamped to >= 1
|
||||
kv_indices_ptr, # out: ragged ids, one run per request
|
||||
table_stride: tl.constexpr,
|
||||
Q_LEN: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Rectangular token table -> ragged (kv_indices, kv_indptr).
|
||||
|
||||
Every row of a request repeats that request's shard, so this reads row 0.
|
||||
"""
|
||||
# The grid is (bs,) and the token loop is inside the kernel, so a cuda
|
||||
# graph can capture this launch.
|
||||
req = tl.program_id(0)
|
||||
out_offset = tl.load(kv_indptr_ptr + req)
|
||||
local_len = tl.load(local_kv_lens_ptr + req).to(tl.int32)
|
||||
# The table must be wide enough for the whole shard. A narrow table makes
|
||||
# this kernel read the next query row and return the wrong ids.
|
||||
tl.device_assert(local_len <= table_stride)
|
||||
|
||||
if local_len == 0:
|
||||
# Give an empty shard one slot, because the attention kernel fails
|
||||
# on a shard of length zero. The caller removes that row later.
|
||||
tl.store(kv_indices_ptr + out_offset, 0)
|
||||
else:
|
||||
row = token_table_ptr + req * Q_LEN * table_stride
|
||||
for i in range(tl.cdiv(local_len, BLOCK_SIZE)):
|
||||
offset = tl.arange(0, BLOCK_SIZE).to(tl.int64) + i * BLOCK_SIZE
|
||||
mask = offset < local_len
|
||||
tl.store(
|
||||
kv_indices_ptr + out_offset + offset,
|
||||
tl.load(row + offset, mask=mask),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def create_dcp_kv_indices(
|
||||
kv_indptr,
|
||||
|
||||
@@ -193,6 +193,10 @@ class EnvIntWithAlias(_DeprecatedEnvFallback, EnvInt):
|
||||
pass
|
||||
|
||||
|
||||
class EnvStrWithAlias(_DeprecatedEnvFallback, EnvStr):
|
||||
pass
|
||||
|
||||
|
||||
class EnvFloat(EnvField):
|
||||
def parse(self, value: str) -> float:
|
||||
try:
|
||||
@@ -964,10 +968,14 @@ class Envs:
|
||||
# import and Triton cga_layout prerequisites hold. Set to 0 to force the
|
||||
# zero-pad mla_decode_fwd fallback (benchmarking / emergency disable).
|
||||
SGLANG_AITER_MLA_GLUON = EnvBool(True)
|
||||
# Select the AITER MLA kernel for DCP decode. "gluon" preserves the native
|
||||
# low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and
|
||||
# runs mla_decode_fwd with persistent metadata and return_lse=True.
|
||||
SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon")
|
||||
# Select the AITER MLA kernel for decode, "asm" or "gluon".
|
||||
SGLANG_AITER_MLA_DECODE_BACKEND = EnvStrWithAlias(
|
||||
"asm", deprecated_name="SGLANG_AITER_MLA_DCP_DECODE_BACKEND"
|
||||
)
|
||||
# Select the AITER MLA kernel for target verify, "asm" or "gluon".
|
||||
SGLANG_AITER_MLA_VERIFY_BACKEND = EnvStr("asm")
|
||||
# Let aiter plan the KV splits for the asm persistent MLA decode.
|
||||
SGLANG_AITER_MLA_AUTO_KV_SPLITS = EnvBool(False)
|
||||
|
||||
# DSV4 Aiter flags
|
||||
SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False)
|
||||
|
||||
@@ -72,7 +72,10 @@ except ImportError:
|
||||
"aiter is AMD specific kernel library. Please make sure aiter is installed on your AMD device."
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.attention.dcp_kernels import create_mla_kv_page_table_for_dcp
|
||||
from sglang.kernels.ops.attention.dcp_kernels import (
|
||||
compact_dcp_verify_token_table_to_ragged,
|
||||
create_mla_kv_page_table_for_dcp,
|
||||
)
|
||||
from sglang.kernels.ops.attention.merge_state import merge_state_triton
|
||||
from sglang.kernels.ops.attention.utils import (
|
||||
launch_reshape_and_cache_flash,
|
||||
@@ -104,6 +107,37 @@ _use_fp8_prefill_attn = (
|
||||
get_bool_env_var("SGLANG_AITER_FP8_PREFILL_ATTN", "True") and is_gfx95_supported()
|
||||
)
|
||||
|
||||
# Max qlen limitation of the asm ps kernels to handle q with bf16 dtype
|
||||
_MLA_ASM_PS_MAX_QLEN = 4
|
||||
|
||||
# (query head count -> query lengths) at which aiter's scheduler treats the
|
||||
# count as native, on gfx950 with an fp8 query and KV. None means any length.
|
||||
# This map is copied from natively_supported in aiter/ops/attention.py. See
|
||||
# https://github.com/ROCm/aiter/blob/bf37db00749722ae29ac27c101f534da43c586f5/aiter/ops/attention.py#L1669-L1706
|
||||
_MLA_ASM_CPRR_NATIVE_QLENS = {
|
||||
16: None,
|
||||
32: frozenset({4}),
|
||||
96: frozenset(range(1, 6 + 1)),
|
||||
128: None,
|
||||
}
|
||||
|
||||
|
||||
def _asm_cprr_supports_shape(heads: int, q_len: int) -> bool:
|
||||
"""Whether aiter schedules this head count natively at this query length."""
|
||||
if heads not in _MLA_ASM_CPRR_NATIVE_QLENS:
|
||||
return False
|
||||
allowed = _MLA_ASM_CPRR_NATIVE_QLENS[heads]
|
||||
return allowed is None or q_len in allowed
|
||||
|
||||
|
||||
def _asm_cprr_kernel_heads(gathered_heads: int, q_len: int) -> int:
|
||||
"""The smallest supported head count at or above the gathered one, or 0."""
|
||||
for heads in sorted(_MLA_ASM_CPRR_NATIVE_QLENS):
|
||||
if heads >= gathered_heads and _asm_cprr_supports_shape(heads, q_len):
|
||||
return heads
|
||||
return 0
|
||||
|
||||
|
||||
# (v_head_dim -> query head counts) that aiter's mla_reduce_v1 has an
|
||||
# instantiation. This map is copied from MLA_REDUCE_ROUTER in
|
||||
# aiter/csrc/kernels/mla/reduce.cu. See https://github.com/ROCm/aiter/blob/7915f53a4225b3f9cb632a97a23369dbebcf1be0/csrc/kernels/mla/reduce.cu#L923
|
||||
@@ -186,6 +220,13 @@ class ForwardMetadata:
|
||||
prefill_ps_metadata: Optional[MlaPrefillPsMetadata] = None
|
||||
chunked_skip_prefix_ps_metadata: Optional[MlaPrefillPsMetadata] = None
|
||||
chunked_prefix_ps_metadatas: Optional[list[Optional[MlaPrefillPsMetadata]]] = None
|
||||
# cprr
|
||||
global_kv_indptr: Optional[torch.Tensor] = None
|
||||
use_asm_cprr_verify: bool = False
|
||||
cprr_kernel_heads: int = 0
|
||||
cprr_kv_indices: Optional[torch.Tensor] = None
|
||||
cprr_kv_indptr: Optional[torch.Tensor] = None
|
||||
cprr_empty_rows: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def _build_paged_kv_view(
|
||||
@@ -313,6 +354,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# can never carry more seqs than the pool.
|
||||
extend_dummy_seqs_capped_by_req_pool: bool = True
|
||||
|
||||
# declare here to avoid CI failure in test_aiter_fp8_q_unified_attention.py
|
||||
use_mla_auto_kv_splits: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
@@ -350,18 +394,33 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
|
||||
|
||||
self.dcp_world_size = get_parallel().attn_dcp_size
|
||||
self.mla_dcp_decode_backend = (
|
||||
envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower()
|
||||
)
|
||||
if self.mla_dcp_decode_backend not in ("gluon", "asm"):
|
||||
self.mla_decode_backend = envs.SGLANG_AITER_MLA_DECODE_BACKEND.get().lower()
|
||||
if self.mla_decode_backend not in ("gluon", "asm"):
|
||||
raise ValueError(
|
||||
"SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', "
|
||||
f"got {self.mla_dcp_decode_backend!r}"
|
||||
"SGLANG_AITER_MLA_DECODE_BACKEND must be 'gluon' or 'asm', "
|
||||
f"got {self.mla_decode_backend!r}"
|
||||
)
|
||||
self.use_mla_dcp_asm = (
|
||||
self.use_mla
|
||||
and self.dcp_world_size > 1
|
||||
and self.mla_dcp_decode_backend == "asm"
|
||||
and self.mla_decode_backend == "asm"
|
||||
)
|
||||
self.mla_verify_backend = envs.SGLANG_AITER_MLA_VERIFY_BACKEND.get().lower()
|
||||
if self.mla_verify_backend not in ("gluon", "asm"):
|
||||
raise ValueError(
|
||||
"SGLANG_AITER_MLA_VERIFY_BACKEND must be 'gluon' or 'asm', "
|
||||
f"got {self.mla_verify_backend!r}"
|
||||
)
|
||||
self._cprr_launch_logged = False
|
||||
# fast_mode / intra_batch_mode for the cp verify schedule. Not the
|
||||
# module defaults: those size reduce_partial_map at 65536 entries
|
||||
# instead of 1276, and mla_decode_fwd then asks for a 64 GiB logits
|
||||
# buffer.
|
||||
self.cprr_fast_mode = True
|
||||
self.cprr_intra_batch_mode = False
|
||||
gathered_heads = self.num_head * self.dcp_world_size
|
||||
self.cprr_kernel_heads = _asm_cprr_kernel_heads(
|
||||
gathered_heads, self.num_draft_tokens or 1
|
||||
)
|
||||
|
||||
# Get v_head_dim based on model type
|
||||
@@ -570,10 +629,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
global _use_mla_ps_kernel, fast_mode, intra_batch_mode
|
||||
|
||||
# current mla_decode_fwd only support fake-nps in num_head == 16
|
||||
# so all num_head size does not use qh16 kernel to simulate
|
||||
# it should not use fake-nps (fast_mode = False, intra_batch_mode = True)
|
||||
# it will cause gpu-fault or accuracy issue.
|
||||
# fake-nps (fast_mode False, intra_batch_mode True) picks aiter's
|
||||
# v1_0 scheduler, which folds these counts to 16 while the gfx950
|
||||
# fp8 launch does not. Both sides must fold alike or the kernel
|
||||
# faults. At 16 neither folds, so fake-nps is correct there.
|
||||
if self.mla_kernel_num_head_padded in (32, 64, 128):
|
||||
fast_mode = True
|
||||
intra_batch_mode = False
|
||||
@@ -592,14 +651,16 @@ class AiterAttnBackend(AttentionBackend):
|
||||
_use_mla_ps_kernel = False
|
||||
fast_mode = False
|
||||
intra_batch_mode = False
|
||||
# Zero-pad topology (h12->qh16): prefer Gluon decode over PS kernel.
|
||||
if self.head_pad_mode == "zero" and self.kv_cache_dtype == fp8_dtype:
|
||||
# Disable ps only when gluon kernel is selected to avoid falling
|
||||
# back to incorrect aiter kernel
|
||||
if prefer_mla_gluon_decode(
|
||||
head_pad_mode=self.head_pad_mode,
|
||||
num_head=self.num_head,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
if (
|
||||
prefer_mla_gluon_decode(
|
||||
head_pad_mode=self.head_pad_mode,
|
||||
num_head=self.num_head,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
and not self._asm_ps_supports_decode_and_verify()
|
||||
):
|
||||
_use_mla_ps_kernel = False
|
||||
fast_mode = False
|
||||
@@ -612,6 +673,27 @@ class AiterAttnBackend(AttentionBackend):
|
||||
if self.num_draft_tokens is None and use_any_mla_persist:
|
||||
self.max_split_per_batch = 64
|
||||
|
||||
# When the env var is on, aiter plans the KV splits. The default
|
||||
# scheduler gives every request ceil(num_cu / batch_size) splits.
|
||||
# It ignores the KV length. That clamp is inside the kernel, so no
|
||||
# host value lifts it. The v1_2 scheduler sizes the count from the
|
||||
# workload. A max_split_per_batch of -1 selects its auto branch.
|
||||
#
|
||||
# Under DCP the cp verify route already runs the v1_2 scheduler:
|
||||
# make_mla_decode_meta_data_buffer takes cprr_fast_mode and
|
||||
# cprr_intra_batch_mode instead. There only the -1 has an effect.
|
||||
self.use_mla_auto_kv_splits = (
|
||||
use_any_mla_persist and envs.SGLANG_AITER_MLA_AUTO_KV_SPLITS.get()
|
||||
)
|
||||
if self.use_mla_auto_kv_splits:
|
||||
logger.info(
|
||||
"aiter MLA: aiter plans the KV splits "
|
||||
"(SGLANG_AITER_MLA_AUTO_KV_SPLITS=1)"
|
||||
)
|
||||
fast_mode = True
|
||||
intra_batch_mode = False
|
||||
self.max_split_per_batch = -1
|
||||
|
||||
self.fix_max_split_per_batch = self.max_split_per_batch
|
||||
|
||||
def pad_heads(self, x: torch.Tensor, padded: int) -> torch.Tensor:
|
||||
@@ -688,13 +770,21 @@ class AiterAttnBackend(AttentionBackend):
|
||||
*,
|
||||
metadata_fast_mode: Optional[bool] = None,
|
||||
metadata_intra_batch_mode: Optional[bool] = None,
|
||||
nhead_override: Optional[int] = None,
|
||||
max_split_per_batch: Optional[int] = None,
|
||||
):
|
||||
"""Allocate the PS work-schedule buffers, sized for nhead_override."""
|
||||
# Under DCP this is the gathered head count (num_head * dcp_world_size);
|
||||
# equals num_head_padded when DCP is off.
|
||||
nhead = self.mla_kernel_num_head_padded
|
||||
nhead = (
|
||||
self.mla_kernel_num_head_padded
|
||||
if nhead_override is None
|
||||
else nhead_override
|
||||
)
|
||||
dtype = self.kv_cache_dtype
|
||||
|
||||
if self.attn_dp_enabled:
|
||||
# The auto count is -1, which this min() would turn into a real cap.
|
||||
if self.attn_dp_enabled and not self.use_mla_auto_kv_splits:
|
||||
gpu = torch.cuda.current_device()
|
||||
device_properties = torch.cuda.get_device_properties(gpu)
|
||||
cu_num = device_properties.multi_processor_count
|
||||
@@ -728,6 +818,11 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fast_mode=metadata_fast_mode,
|
||||
num_kv_splits=self.max_split_per_batch,
|
||||
intra_batch_mode=metadata_intra_batch_mode,
|
||||
**(
|
||||
{}
|
||||
if max_split_per_batch is None
|
||||
else {"max_split_per_batch": max_split_per_batch}
|
||||
),
|
||||
)
|
||||
|
||||
# aiter implementation
|
||||
@@ -777,16 +872,23 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fast_mode,
|
||||
max_split_per_batch,
|
||||
intra_batch_mode,
|
||||
is_cp_round_robin=False,
|
||||
nhead_override=None,
|
||||
):
|
||||
nhead_kv = 1
|
||||
page_size = self.page_size
|
||||
dtype = self.kv_cache_dtype
|
||||
nhead = (
|
||||
self.mla_kernel_num_head_padded
|
||||
if nhead_override is None
|
||||
else nhead_override
|
||||
)
|
||||
|
||||
meta = get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_last_page_len,
|
||||
self.mla_kernel_num_head_padded // nhead_kv,
|
||||
nhead // nhead_kv,
|
||||
nhead_kv,
|
||||
False,
|
||||
work_metadata,
|
||||
@@ -801,6 +903,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fast_mode=fast_mode,
|
||||
max_split_per_batch=max_split_per_batch,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
is_cp_round_robin=is_cp_round_robin,
|
||||
dtype_q=dtype,
|
||||
dtype_kv=dtype,
|
||||
)
|
||||
@@ -1287,6 +1390,52 @@ class AiterAttnBackend(AttentionBackend):
|
||||
f"Got topk={self.topk}."
|
||||
)
|
||||
|
||||
def _asm_ps_supports_decode_and_verify(self) -> bool:
|
||||
"""Whether the asm PS MLA kernels support both decode and target verify."""
|
||||
verify_q_len = self.num_draft_tokens or 1
|
||||
if self.dcp_world_size > 1:
|
||||
if _asm_cprr_supports_shape(self.cprr_kernel_heads, verify_q_len):
|
||||
return True
|
||||
logger.info(
|
||||
"aiter MLA decode and verify stay on the Gluon kernel: the cp "
|
||||
"verify kernel supports no %d-token window at %d gathered heads.",
|
||||
verify_q_len,
|
||||
self.num_head * self.dcp_world_size,
|
||||
)
|
||||
return False
|
||||
if self._asm_ps_supports_qlen(verify_q_len):
|
||||
return True
|
||||
logger.info(
|
||||
"aiter MLA decode and verify stay on the Gluon kernel: target "
|
||||
"verify is %d tokens wide and the asm kernels support at most %d at "
|
||||
"%d heads. A DSPARK block size of %d or lower would fit.",
|
||||
verify_q_len,
|
||||
_MLA_ASM_PS_MAX_QLEN,
|
||||
self.mla_kernel_num_head_padded,
|
||||
_MLA_ASM_PS_MAX_QLEN - 1,
|
||||
)
|
||||
return False
|
||||
|
||||
def _asm_ps_supports_qlen(self, q_len: int) -> bool:
|
||||
"""Whether the asm PS dispatch has a kernel for this query length."""
|
||||
return q_len <= _MLA_ASM_PS_MAX_QLEN or (
|
||||
self.kv_cache_dtype == fp8_dtype
|
||||
and self.mla_kernel_num_head_padded == 16
|
||||
and is_gfx95_supported()
|
||||
)
|
||||
|
||||
def _asm_cprr_supports_verify_shape(self, q_len: Optional[int]) -> bool:
|
||||
"""Whether this topology has an asm cp kernel for DCP target verify."""
|
||||
return (
|
||||
_use_mla_ps_kernel
|
||||
and self.mla_verify_backend == "asm"
|
||||
and self.dcp_world_size > 1
|
||||
and self.kv_cache_dtype == fp8_dtype
|
||||
and q_len is not None
|
||||
and self.cprr_kernel_heads > 0
|
||||
and _asm_cprr_supports_shape(self.cprr_kernel_heads, q_len)
|
||||
)
|
||||
|
||||
def _mla_decode_fwd_with_head_pad(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -1301,6 +1450,27 @@ class AiterAttnBackend(AttentionBackend):
|
||||
q / o must already be shaped (..., num_head, head_dim).
|
||||
"""
|
||||
num_head = layer.tp_q_head_num
|
||||
if (
|
||||
_use_mla_ps_kernel
|
||||
and self.dcp_world_size <= 1
|
||||
and self._asm_ps_supports_qlen(kwargs.get("max_seqlen_q") or 1)
|
||||
):
|
||||
# The asm kernel reads KV row 0 for masked positions and weights it
|
||||
# by p = 0. A NaN there spreads to every request, because
|
||||
# 0 * NaN = NaN. Row 0 is where CUDA-graph padding tokens write
|
||||
# their KV, and that KV can be NaN. Gluon, the kernel this path
|
||||
# replaces, does not read row 0.
|
||||
k_buffer_flat[0].zero_()
|
||||
if (
|
||||
self.kv_cache_dtype == fp8_dtype
|
||||
and self.mla_kernel_num_head_padded == 16
|
||||
and is_gfx95_supported()
|
||||
and (kwargs.get("max_seqlen_q") or 1) > _MLA_ASM_PS_MAX_QLEN
|
||||
and q.dtype != fp8_dtype
|
||||
):
|
||||
q_flat = q.reshape(q.shape[0], -1).to(fp8_dtype)
|
||||
kwargs["q_scale"] = torch.ones((), dtype=torch.float32, device=q.device)
|
||||
q = q_flat.view(q.shape)
|
||||
if self.head_pad_mode == "repeat" or (
|
||||
self.head_pad_mode == "none" and self.num_head_padded != self.num_head
|
||||
):
|
||||
@@ -1367,6 +1537,17 @@ class AiterAttnBackend(AttentionBackend):
|
||||
return 1
|
||||
return int(seq_lens.max().item())
|
||||
|
||||
def _kernel_num_kv_splits(self) -> Optional[int]:
|
||||
"""The split count for a decode kernel call, None to let aiter pick.
|
||||
|
||||
The metadata call takes an int and gets -1 for the auto branch. The
|
||||
kernel takes None instead. mla.py re-decides persistent mode only on
|
||||
None. It would read -1 as a real count.
|
||||
"""
|
||||
if self.use_mla_auto_kv_splits:
|
||||
return None
|
||||
return self.forward_metadata.num_kv_splits
|
||||
|
||||
def _forward_mla_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -1382,6 +1563,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
head_pad_mode=getattr(self, "head_pad_mode", "none"),
|
||||
num_head=getattr(self, "num_head", layer.tp_q_head_num),
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
) and not (
|
||||
_use_mla_ps_kernel
|
||||
and self.mla_decode_backend == "asm"
|
||||
and self._asm_ps_supports_qlen(max_q_len)
|
||||
):
|
||||
return mla_gluon_decode(
|
||||
q=q,
|
||||
@@ -1401,7 +1586,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_indptr = self.forward_metadata.reduce_indptr
|
||||
reduce_final_map = self.forward_metadata.reduce_final_map
|
||||
reduce_partial_map = self.forward_metadata.reduce_partial_map
|
||||
num_kv_splits = self.forward_metadata.num_kv_splits
|
||||
num_kv_splits = self._kernel_num_kv_splits()
|
||||
|
||||
return self._mla_decode_fwd_with_head_pad(
|
||||
q,
|
||||
@@ -1426,10 +1611,17 @@ class AiterAttnBackend(AttentionBackend):
|
||||
num_kv_splits=num_kv_splits,
|
||||
)
|
||||
|
||||
def _get_dcp_graph_max_local_kv_len(self) -> int:
|
||||
"""Static upper bound on this rank's shard, ceil(max_context_len / W)."""
|
||||
def _dcp_max_local_kv_len(self, max_global_kv_len: int) -> int:
|
||||
"""Widest shard one rank can hold for a global kv length."""
|
||||
w = max(self.dcp_world_size, 1)
|
||||
return (self.max_context_len + w - 1) // w
|
||||
return (max_global_kv_len + w - 1) // w
|
||||
|
||||
def _dcp_graph_max_local_kv_len(self) -> int:
|
||||
"""Static upper bound on this rank's shard under a cuda graph."""
|
||||
max_global_kv_len = self.max_context_len
|
||||
if self._asm_cprr_supports_verify_shape(self.num_draft_tokens):
|
||||
max_global_kv_len += self.num_draft_tokens
|
||||
return self._dcp_max_local_kv_len(max_global_kv_len)
|
||||
|
||||
def _forward_decode_dcp(self, q, k_buffer, layer, k_descale):
|
||||
"""Attend this rank's KV shard for decode -> (out, natural-log lse)."""
|
||||
@@ -1446,8 +1638,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
q_mla = q.view(bs, num_heads, layer.qk_head_dim)
|
||||
q_scale = k_descale
|
||||
if q_mla.dtype != fp8_dtype:
|
||||
q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1))
|
||||
q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim)
|
||||
q_mla = (
|
||||
q_mla.reshape(bs, -1)
|
||||
.to(fp8_dtype)
|
||||
.view(bs, num_heads, layer.qk_head_dim)
|
||||
)
|
||||
q_scale = torch.ones((), dtype=torch.float32, device=q.device)
|
||||
|
||||
out = torch.empty(
|
||||
(bs, num_heads, layer.v_head_dim),
|
||||
@@ -1465,7 +1661,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fm.max_q_len or 1,
|
||||
sm_scale=layer.scaling,
|
||||
logit_cap=layer.logit_cap,
|
||||
num_kv_splits=fm.num_kv_splits,
|
||||
num_kv_splits=self._kernel_num_kv_splits(),
|
||||
work_meta_data=fm.work_metadata,
|
||||
work_indptr=fm.work_indptr,
|
||||
work_info_set=fm.work_info_set,
|
||||
@@ -1496,13 +1692,126 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
return out, lse.view(bs, num_heads)
|
||||
|
||||
def _forward_verify_dcp(self, q, k_window, layer, k_descale):
|
||||
"""Attend the committed prefix and the verify window separately, then
|
||||
merge -> (out, natural-log lse).
|
||||
def _forward_verify_asm_cprr(self, q, layer, k_descale, n_rows):
|
||||
"""DCP target verify on the asm cp round-robin kernel: one masked pass."""
|
||||
fm = self.forward_metadata
|
||||
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
|
||||
kernel_heads = fm.cprr_kernel_heads or num_heads
|
||||
if not self._cprr_launch_logged:
|
||||
self._cprr_launch_logged = True
|
||||
logger.info(
|
||||
"aiter DCP cp verify: gathered=%d kernel_heads=%d%s "
|
||||
"intra_batch=%s max_q_len=%d reduce_partial_map=%d",
|
||||
num_heads,
|
||||
kernel_heads,
|
||||
" (padded)" if kernel_heads != num_heads else " (native)",
|
||||
self.cprr_intra_batch_mode,
|
||||
fm.max_q_len,
|
||||
fm.reduce_partial_map.numel(),
|
||||
)
|
||||
# torch.full, not torch.as_tensor: as_tensor copies from host memory,
|
||||
# which a cuda graph capture cannot record.
|
||||
kv_scale = torch.full(
|
||||
(),
|
||||
self._resolve_fp8_kv_scale_float(layer, k_descale),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
kv_indices, kv_indptr = fm.cprr_kv_indices, fm.cprr_kv_indptr
|
||||
q_in = (
|
||||
q.view(n_rows, -1).to(fp8_dtype).view(n_rows, num_heads, layer.qk_head_dim)
|
||||
)
|
||||
q_scale = torch.ones((), dtype=torch.float32, device=q.device)
|
||||
if kernel_heads != num_heads:
|
||||
q_in = self.pad_heads(q_in, kernel_heads)
|
||||
out = q.new_empty(
|
||||
(n_rows, kernel_heads, layer.v_head_dim), dtype=self.input_dtype
|
||||
)
|
||||
_, lse = mla_decode_fwd(
|
||||
q_in,
|
||||
self.token_to_kv_pool.get_key_buffer(layer.layer_id).view(
|
||||
-1, 1, 1, layer.qk_head_dim
|
||||
),
|
||||
out,
|
||||
fm.qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
fm.kv_last_page_len,
|
||||
fm.max_q_len,
|
||||
sm_scale=layer.scaling,
|
||||
work_meta_data=fm.work_metadata,
|
||||
work_indptr=fm.work_indptr,
|
||||
work_info_set=fm.work_info_set,
|
||||
reduce_indptr=fm.reduce_indptr,
|
||||
reduce_final_map=fm.reduce_final_map,
|
||||
reduce_partial_map=fm.reduce_partial_map,
|
||||
q_scale=q_scale,
|
||||
kv_scale=kv_scale,
|
||||
intra_batch_mode=self.cprr_intra_batch_mode,
|
||||
num_kv_splits=self._kernel_num_kv_splits(),
|
||||
return_lse=True,
|
||||
g_kv_indptr=fm.global_kv_indptr,
|
||||
cp_world_size=self.dcp_world_size,
|
||||
cp_rank=get_parallel().attn_dcp_rank,
|
||||
causal=True,
|
||||
)
|
||||
lse = lse.view(n_rows, kernel_heads)
|
||||
if kernel_heads != num_heads:
|
||||
out = out[:, :num_heads, :]
|
||||
lse = lse[:, :num_heads]
|
||||
return out, lse
|
||||
|
||||
Splitting at the window boundary avoids the one thing the decode kernel
|
||||
cannot do under DCP: mask on the GLOBAL position g(j) = j * W + r.
|
||||
"""
|
||||
def _build_dcp_verify_ragged_indices(
|
||||
self,
|
||||
verify_token_table: torch.Tensor,
|
||||
local_kv_lens: torch.Tensor,
|
||||
bs: int,
|
||||
q_len: int,
|
||||
out_indices: Optional[torch.Tensor] = None,
|
||||
out_indptr: Optional[torch.Tensor] = None,
|
||||
out_empty: Optional[torch.Tensor] = None,
|
||||
):
|
||||
"""This rank's verify shard as ragged (kv_indices, kv_indptr, empty_rows)."""
|
||||
# Pass out_* to use the address-stable buffers of the cuda graph.
|
||||
# Omit them to allocate new tensors.
|
||||
# The .contiguous() call is necessary. The slice has stride q_len, and
|
||||
# .to() returns an int32 tensor unchanged, so without the copy the
|
||||
# kernel reads lens[req] instead of lens[req * q_len]. The two agree
|
||||
# only at batch size 1.
|
||||
lens = local_kv_lens[: bs * q_len : q_len].to(torch.int32).contiguous()
|
||||
# The kernel faults on a zero-length shard, so give every shard a slot.
|
||||
clamped = lens.clamp_min(1)
|
||||
if out_indptr is None:
|
||||
kv_indptr = torch.zeros(bs + 1, dtype=torch.int32, device=lens.device)
|
||||
else:
|
||||
kv_indptr = out_indptr[: bs + 1]
|
||||
# Not kv_indptr[0] = 0: that copies from host memory, which a
|
||||
# cuda graph capture cannot record.
|
||||
kv_indptr[:1].zero_()
|
||||
torch.cumsum(clamped, dim=0, out=kv_indptr[1:])
|
||||
|
||||
if out_indices is None:
|
||||
kv_indices = verify_token_table.new_empty(
|
||||
(int(clamped.sum().item()),), dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
kv_indices = out_indices
|
||||
compact_dcp_verify_token_table_to_ragged[(bs,)](
|
||||
verify_token_table,
|
||||
lens,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
table_stride=verify_token_table.shape[1],
|
||||
Q_LEN=q_len,
|
||||
BLOCK_SIZE=_DCP_VERIFY_TABLE_COLS_PER_BLOCK,
|
||||
)
|
||||
empty_rows = local_kv_lens[: bs * q_len].eq(0)
|
||||
if out_empty is not None:
|
||||
empty_rows = out_empty[: bs * q_len].copy_(empty_rows)
|
||||
return kv_indices, kv_indptr, empty_rows
|
||||
|
||||
def _forward_verify_dcp(self, q, k_window, layer, k_descale):
|
||||
"""DCP target verify -> (out, natural-log lse) for the cross-rank merge."""
|
||||
fm = self.forward_metadata
|
||||
q_len = fm.max_q_len
|
||||
num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp
|
||||
@@ -1510,6 +1819,13 @@ class AiterAttnBackend(AttentionBackend):
|
||||
n_rows = seqused_k.shape[0]
|
||||
bs = n_rows // q_len
|
||||
|
||||
if fm.use_asm_cprr_verify:
|
||||
out_a, lse_a = self._forward_verify_asm_cprr(q, layer, k_descale, n_rows)
|
||||
empty = fm.cprr_empty_rows
|
||||
out_a.masked_fill_(empty[:, None, None], 0.0)
|
||||
lse_a.masked_fill_(empty[:, None], -1e30)
|
||||
return out_a, lse_a
|
||||
|
||||
out_a, lse_a = mla_gluon_decode(
|
||||
q=q.view(n_rows, num_heads, layer.qk_head_dim),
|
||||
k_buffer=self.token_to_kv_pool.get_key_buffer(layer.layer_id),
|
||||
@@ -1730,6 +2046,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# dcp metadata
|
||||
local_kv_lens = None
|
||||
verify_token_table = None
|
||||
global_kv_indptr = None
|
||||
use_asm_cprr_verify = False
|
||||
cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None
|
||||
if forward_batch.forward_mode.is_decode_or_idle():
|
||||
if spec_info is None or forward_batch.forward_mode.is_idle():
|
||||
kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0)
|
||||
@@ -1967,7 +2286,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
if self.use_mla:
|
||||
draft_num = spec_info.draft_token_num
|
||||
device = forward_batch.seq_lens.device
|
||||
if self.dcp_world_size > 1:
|
||||
use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(draft_num)
|
||||
if self.dcp_world_size > 1 and not use_asm_cprr_verify:
|
||||
kv_lens = forward_batch.seq_lens.to(torch.int32).clone()
|
||||
kv_lens_sum = forward_batch.seq_lens_sum
|
||||
else:
|
||||
@@ -2001,6 +2321,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
)
|
||||
|
||||
if self.dcp_world_size > 1:
|
||||
global_kv_indptr = kv_indptr.clone()
|
||||
self._plan_dcp_decode_metadata(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
@@ -2016,11 +2337,27 @@ class AiterAttnBackend(AttentionBackend):
|
||||
forward_batch.req_pool_indices,
|
||||
bs,
|
||||
draft_num,
|
||||
(max_kv_len + self.dcp_world_size - 1) // self.dcp_world_size,
|
||||
self._dcp_max_local_kv_len(
|
||||
max_kv_len + (draft_num if use_asm_cprr_verify else 0)
|
||||
),
|
||||
)
|
||||
|
||||
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
|
||||
if use_asm_cprr_verify:
|
||||
(
|
||||
cprr_kv_indices,
|
||||
cprr_kv_indptr,
|
||||
cprr_empty_rows,
|
||||
) = self._build_dcp_verify_ragged_indices(
|
||||
verify_token_table, local_kv_lens, bs, draft_num
|
||||
)
|
||||
if _use_mla_ps_kernel and (
|
||||
self.dcp_world_size <= 1 or use_asm_cprr_verify
|
||||
):
|
||||
max_seqlen_qo = draft_num
|
||||
is_cp_round_robin = self.dcp_world_size > 1
|
||||
metadata_heads = (
|
||||
self.cprr_kernel_heads if use_asm_cprr_verify else None
|
||||
)
|
||||
(
|
||||
work_metadata,
|
||||
work_indptr,
|
||||
@@ -2028,13 +2365,26 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_indptr,
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, bs)
|
||||
) = self.make_mla_decode_meta_data_buffer(
|
||||
max_seqlen_qo,
|
||||
bs,
|
||||
metadata_fast_mode=(
|
||||
self.cprr_fast_mode if use_asm_cprr_verify else None
|
||||
),
|
||||
metadata_intra_batch_mode=(
|
||||
self.cprr_intra_batch_mode if use_asm_cprr_verify else None
|
||||
),
|
||||
nhead_override=metadata_heads,
|
||||
max_split_per_batch=(
|
||||
self.max_split_per_batch if use_asm_cprr_verify else None
|
||||
),
|
||||
)
|
||||
|
||||
num_kv_splits = self.max_split_per_batch
|
||||
|
||||
self.make_mla_meta_data(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
cprr_kv_indptr if use_asm_cprr_verify else kv_indptr,
|
||||
self.kv_last_page_len[:bs],
|
||||
work_metadata,
|
||||
work_info_set,
|
||||
@@ -2043,9 +2393,17 @@ class AiterAttnBackend(AttentionBackend):
|
||||
reduce_final_map,
|
||||
reduce_partial_map,
|
||||
max_seqlen_qo,
|
||||
fast_mode=fast_mode,
|
||||
fast_mode=(
|
||||
self.cprr_fast_mode if use_asm_cprr_verify else fast_mode
|
||||
),
|
||||
max_split_per_batch=num_kv_splits,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
intra_batch_mode=(
|
||||
self.cprr_intra_batch_mode
|
||||
if use_asm_cprr_verify
|
||||
else intra_batch_mode
|
||||
),
|
||||
is_cp_round_robin=is_cp_round_robin,
|
||||
nhead_override=metadata_heads,
|
||||
)
|
||||
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
@@ -2066,6 +2424,14 @@ class AiterAttnBackend(AttentionBackend):
|
||||
run_graph=False,
|
||||
local_kv_lens=local_kv_lens,
|
||||
verify_token_table=verify_token_table,
|
||||
global_kv_indptr=global_kv_indptr,
|
||||
use_asm_cprr_verify=use_asm_cprr_verify,
|
||||
cprr_kernel_heads=self.cprr_kernel_heads
|
||||
if use_asm_cprr_verify
|
||||
else 0,
|
||||
cprr_kv_indices=cprr_kv_indices,
|
||||
cprr_kv_indptr=cprr_kv_indptr,
|
||||
cprr_empty_rows=cprr_empty_rows,
|
||||
)
|
||||
else:
|
||||
draft_num = forward_batch.input_ids.shape[0] // bs
|
||||
@@ -2395,10 +2761,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.cuda_graph_verify_local_kv_lens = torch.zeros(
|
||||
(n_verify_rows,), dtype=torch.int32, device=self.device
|
||||
)
|
||||
# Capture-stable token table, filled out-of-graph. One column per
|
||||
# TOKEN, so the width is the worst-case shard ceil(ctx_len / W).
|
||||
self.cuda_graph_verify_token_table = torch.zeros(
|
||||
(n_verify_rows, self._get_dcp_graph_max_local_kv_len()),
|
||||
(n_verify_rows, self._dcp_graph_max_local_kv_len()),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
@@ -2406,10 +2770,24 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# verify plan without a sync; see _plan_dcp_decode_metadata.
|
||||
self.cuda_graph_dcp_static_local_kv_lens = torch.full(
|
||||
(max_bs,),
|
||||
self._get_dcp_graph_max_local_kv_len(),
|
||||
self._dcp_graph_max_local_kv_len(),
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
self.cuda_graph_global_kv_indptr = torch.zeros(
|
||||
(max_bs + 1,), dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.cuda_graph_cprr_kv_indptr = torch.zeros(
|
||||
(max_bs + 1,), dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.cuda_graph_cprr_kv_indices = torch.zeros(
|
||||
(max_bs * (self._dcp_graph_max_local_kv_len() + 1),),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.cuda_graph_cprr_empty_rows = torch.zeros(
|
||||
(n_verify_rows,), dtype=torch.bool, device=self.device
|
||||
)
|
||||
if kv_indices_buf is None:
|
||||
max_num_blocks_per_seq = (
|
||||
self.max_context_len + self.page_size - 1
|
||||
@@ -2471,6 +2849,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
(True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode)
|
||||
)
|
||||
|
||||
graph_use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(
|
||||
self.num_draft_tokens
|
||||
)
|
||||
(
|
||||
self.work_metadata,
|
||||
self.work_indptr,
|
||||
@@ -2481,8 +2862,37 @@ class AiterAttnBackend(AttentionBackend):
|
||||
) = self.make_mla_decode_meta_data_buffer(
|
||||
max_seqlen_qo,
|
||||
max_bs,
|
||||
metadata_fast_mode=metadata_fast_mode,
|
||||
metadata_intra_batch_mode=metadata_intra_batch_mode,
|
||||
metadata_fast_mode=(
|
||||
self.cprr_fast_mode
|
||||
if graph_use_asm_cprr_verify
|
||||
else metadata_fast_mode
|
||||
),
|
||||
metadata_intra_batch_mode=(
|
||||
self.cprr_intra_batch_mode
|
||||
if graph_use_asm_cprr_verify
|
||||
else metadata_intra_batch_mode
|
||||
),
|
||||
nhead_override=(
|
||||
self.cprr_kernel_heads if graph_use_asm_cprr_verify else None
|
||||
),
|
||||
max_split_per_batch=(
|
||||
self.max_split_per_batch if graph_use_asm_cprr_verify else None
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"aiter DCP cp verify graph buffers: use_asm_cprr_verify=%s num_draft_tokens=%s "
|
||||
"max_seqlen_qo=%d max_bs=%d nhead=%s intra=%s split_cap=%s "
|
||||
"reduce_partial_map=%d",
|
||||
graph_use_asm_cprr_verify,
|
||||
self.num_draft_tokens,
|
||||
max_seqlen_qo,
|
||||
max_bs,
|
||||
self.cprr_kernel_heads if graph_use_asm_cprr_verify else None,
|
||||
self.cprr_intra_batch_mode
|
||||
if graph_use_asm_cprr_verify
|
||||
else metadata_intra_batch_mode,
|
||||
self.max_split_per_batch if graph_use_asm_cprr_verify else None,
|
||||
self.reduce_partial_map.numel(),
|
||||
)
|
||||
|
||||
else:
|
||||
@@ -2536,6 +2946,9 @@ class AiterAttnBackend(AttentionBackend):
|
||||
# DCP metadata which will be populated for MLA decode when dcp enabled
|
||||
local_kv_lens = None
|
||||
verify_token_table = None
|
||||
global_kv_indptr = None
|
||||
cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None
|
||||
use_asm_cprr_verify = False
|
||||
|
||||
swa_page_table = None
|
||||
# Unified verify never reads this host max (replay sizes from
|
||||
@@ -2712,9 +3125,12 @@ class AiterAttnBackend(AttentionBackend):
|
||||
device=self.device,
|
||||
)
|
||||
if self.use_mla:
|
||||
use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(
|
||||
self.num_draft_tokens
|
||||
)
|
||||
kv_lens = (
|
||||
seq_lens
|
||||
if self.dcp_world_size > 1
|
||||
if (self.dcp_world_size > 1 and not use_asm_cprr_verify)
|
||||
else seq_lens + self.num_draft_tokens
|
||||
)
|
||||
else:
|
||||
@@ -2748,6 +3164,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs]
|
||||
|
||||
if self.use_mla and self.dcp_world_size > 1:
|
||||
global_kv_indptr = self.cuda_graph_global_kv_indptr[: bs + 1]
|
||||
global_kv_indptr.copy_(kv_indptr[: bs + 1])
|
||||
self._plan_dcp_decode_metadata(
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
@@ -2767,19 +3185,36 @@ class AiterAttnBackend(AttentionBackend):
|
||||
req_pool_indices,
|
||||
bs,
|
||||
self.num_draft_tokens,
|
||||
self._get_dcp_graph_max_local_kv_len(),
|
||||
self._dcp_graph_max_local_kv_len(),
|
||||
out=self.cuda_graph_verify_token_table[:n_rows],
|
||||
out_lens=self.cuda_graph_verify_local_kv_lens[:n_rows],
|
||||
)
|
||||
if use_asm_cprr_verify:
|
||||
(
|
||||
cprr_kv_indices,
|
||||
cprr_kv_indptr,
|
||||
cprr_empty_rows,
|
||||
) = self._build_dcp_verify_ragged_indices(
|
||||
verify_token_table,
|
||||
local_kv_lens,
|
||||
bs,
|
||||
self.num_draft_tokens,
|
||||
out_indices=self.cuda_graph_cprr_kv_indices,
|
||||
out_indptr=self.cuda_graph_cprr_kv_indptr,
|
||||
out_empty=self.cuda_graph_cprr_empty_rows,
|
||||
)
|
||||
|
||||
if self.use_mla:
|
||||
max_q_len = self.num_draft_tokens
|
||||
if _use_mla_ps_kernel and self.dcp_world_size <= 1:
|
||||
if _use_mla_ps_kernel and (
|
||||
self.dcp_world_size <= 1 or use_asm_cprr_verify
|
||||
):
|
||||
num_kv_splits = self.max_split_per_batch
|
||||
is_cp_round_robin = self.dcp_world_size > 1
|
||||
|
||||
self.make_mla_meta_data(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
cprr_kv_indptr if use_asm_cprr_verify else kv_indptr,
|
||||
kv_last_page_len,
|
||||
self.work_metadata,
|
||||
self.work_info_set,
|
||||
@@ -2788,9 +3223,19 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.reduce_final_map,
|
||||
self.reduce_partial_map,
|
||||
max_q_len,
|
||||
fast_mode=fast_mode,
|
||||
fast_mode=(
|
||||
self.cprr_fast_mode if use_asm_cprr_verify else fast_mode
|
||||
),
|
||||
max_split_per_batch=num_kv_splits,
|
||||
intra_batch_mode=intra_batch_mode,
|
||||
intra_batch_mode=(
|
||||
self.cprr_intra_batch_mode
|
||||
if use_asm_cprr_verify
|
||||
else intra_batch_mode
|
||||
),
|
||||
is_cp_round_robin=is_cp_round_robin,
|
||||
nhead_override=(
|
||||
self.cprr_kernel_heads if use_asm_cprr_verify else None
|
||||
),
|
||||
)
|
||||
|
||||
work_metadata = self.work_metadata
|
||||
@@ -2817,6 +3262,14 @@ class AiterAttnBackend(AttentionBackend):
|
||||
num_kv_splits=num_kv_splits,
|
||||
local_kv_lens=local_kv_lens,
|
||||
verify_token_table=verify_token_table,
|
||||
global_kv_indptr=global_kv_indptr,
|
||||
use_asm_cprr_verify=use_asm_cprr_verify,
|
||||
cprr_kernel_heads=self.cprr_kernel_heads
|
||||
if use_asm_cprr_verify
|
||||
else 0,
|
||||
cprr_kv_indices=cprr_kv_indices,
|
||||
cprr_kv_indptr=cprr_kv_indptr,
|
||||
cprr_empty_rows=cprr_empty_rows,
|
||||
)
|
||||
else:
|
||||
max_q_len = verify_tokens_per_req
|
||||
@@ -3368,8 +3821,6 @@ class AiterAttnBackend(AttentionBackend):
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
and self.dcp_world_size > 1
|
||||
):
|
||||
# two-stage dcp verify, dispatched before the dims below: the
|
||||
# model provides the per rank kvcache slices.
|
||||
return self._forward_verify_dcp(q, k, layer, k_descale)
|
||||
|
||||
qk_nope_head_dim = k.shape[-1] - qk_rope_head_dim
|
||||
@@ -3534,6 +3985,10 @@ class AiterAttnBackend(AttentionBackend):
|
||||
head_pad_mode=getattr(self, "head_pad_mode", "none"),
|
||||
num_head=getattr(self, "num_head", layer.tp_q_head_num),
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
) and not (
|
||||
_use_mla_ps_kernel
|
||||
and self.mla_verify_backend == "asm"
|
||||
and self._asm_ps_supports_qlen(self.forward_metadata.max_q_len or 1)
|
||||
):
|
||||
return mla_gluon_decode(
|
||||
q=q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
||||
|
||||
@@ -237,6 +237,10 @@ class TestForwardMlaDecodeDispatch(CustomTestCase):
|
||||
be.kv_cache_dtype = fp8_dtype
|
||||
be.head_pad_mode = "zero"
|
||||
be.num_head_padded = 16
|
||||
# The head count the asm kernel runs at. Without DCP it is the padded
|
||||
# per-rank count, so 16 here. _asm_ps_supports_qlen reads it.
|
||||
be.mla_kernel_num_head_padded = 16
|
||||
be.mla_decode_backend = "asm"
|
||||
be.forward_metadata = mock.Mock(
|
||||
max_q_len=max_q_len,
|
||||
kv_indices=torch.zeros(4, dtype=torch.int32),
|
||||
@@ -283,7 +287,9 @@ class TestForwardMlaDecodeDispatch(CustomTestCase):
|
||||
return_value=True,
|
||||
)
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode")
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend._use_mla_ps_kernel", False)
|
||||
def test_uses_gluon_output_when_preferred(self, mock_gluon, _prefer):
|
||||
"""Gluon wins when it is preferred and the asm PS kernel is off."""
|
||||
gluon_out = torch.ones(4, 12, 512)
|
||||
mock_gluon.return_value = gluon_out
|
||||
be = self._make_backend()
|
||||
@@ -302,9 +308,52 @@ class TestForwardMlaDecodeDispatch(CustomTestCase):
|
||||
return_value=True,
|
||||
)
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode")
|
||||
def test_asm_ps_wins_over_preferred_gluon(self, mock_gluon, _prefer):
|
||||
"""The asm PS kernel takes a width it supports, even over preferred Gluon."""
|
||||
be = self._make_backend()
|
||||
be._forward_mla_decode(
|
||||
torch.zeros(4, 12, 576, dtype=torch.bfloat16),
|
||||
_layer(),
|
||||
mock.Mock(),
|
||||
k_descale=1.0,
|
||||
)
|
||||
mock_gluon.assert_not_called()
|
||||
be._mla_decode_fwd_with_head_pad.assert_called_once()
|
||||
|
||||
@mock.patch(
|
||||
"sglang.srt.layers.attention.aiter_backend.prefer_mla_gluon_decode",
|
||||
return_value=True,
|
||||
)
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode")
|
||||
def test_decode_backend_gluon_keeps_gluon(self, mock_gluon, _prefer):
|
||||
"""SGLANG_AITER_MLA_DECODE_BACKEND=gluon holds decode on Gluon."""
|
||||
gluon_out = torch.ones(4, 12, 512)
|
||||
mock_gluon.return_value = gluon_out
|
||||
be = self._make_backend()
|
||||
be.mla_decode_backend = "gluon"
|
||||
out = be._forward_mla_decode(
|
||||
torch.zeros(4, 12, 576, dtype=torch.bfloat16),
|
||||
_layer(),
|
||||
mock.Mock(),
|
||||
k_descale=1.0,
|
||||
)
|
||||
mock_gluon.assert_called_once()
|
||||
be._mla_decode_fwd_with_head_pad.assert_not_called()
|
||||
self.assertIs(out, gluon_out)
|
||||
|
||||
@mock.patch(
|
||||
"sglang.srt.layers.attention.aiter_backend.prefer_mla_gluon_decode",
|
||||
return_value=True,
|
||||
)
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode")
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend._use_mla_ps_kernel", False)
|
||||
def test_passes_max_q_len_as_qlen(self, mock_gluon, _prefer):
|
||||
"""Target-verify must reach the kernel as qlen, not be silently dropped:
|
||||
the ASM fallback cannot serve this topology above qSeqLen 4 at all."""
|
||||
"""Target verify must reach the kernel as qlen, not be silently dropped.
|
||||
|
||||
The asm PS kernel is off here, so the width is Gluon's to serve. With
|
||||
it on, a bf16 query stops at qSeqLen 4 and an fp8 query goes above it;
|
||||
_asm_ps_supports_qlen owns that decision.
|
||||
"""
|
||||
mock_gluon.return_value = torch.ones(32, 12, 512)
|
||||
be = self._make_backend(max_q_len=8)
|
||||
be._forward_mla_decode(
|
||||
@@ -353,16 +402,18 @@ class TestMlaDcpAsmDecode(CustomTestCase):
|
||||
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.mla_decode_fwd", create=True)
|
||||
@mock.patch("sglang.srt.layers.attention.aiter_backend.scaled_fp8_quant")
|
||||
def test_asm_quantizes_gathered_q_and_returns_lse(self, mock_quant, mock_mla):
|
||||
def test_asm_casts_gathered_q_and_returns_lse(self, mock_quant, mock_mla):
|
||||
"""Q reaches the kernel as a plain fp8 cast at a scale of 1.0.
|
||||
|
||||
Scaling by the tensor maximum first is no more accurate on this query
|
||||
and costs about twice the time, so scaled_fp8_quant is not called.
|
||||
"""
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype
|
||||
|
||||
be = self._make_backend()
|
||||
layer = self._make_layer()
|
||||
q = torch.zeros(2, 96, 576, dtype=torch.bfloat16)
|
||||
q_fp8 = torch.zeros(2, 96 * 576, dtype=fp8_dtype)
|
||||
q_scale = torch.tensor([0.25], dtype=torch.float32)
|
||||
lse = torch.arange(2 * 96, dtype=torch.float32).view(2, 96)
|
||||
mock_quant.return_value = (q_fp8, q_scale)
|
||||
mock_mla.return_value = (None, lse)
|
||||
|
||||
out, actual_lse = be._forward_decode_dcp(
|
||||
@@ -376,7 +427,8 @@ class TestMlaDcpAsmDecode(CustomTestCase):
|
||||
self.assertEqual(out.dtype, torch.bfloat16)
|
||||
self.assertTrue(torch.equal(actual_lse, lse))
|
||||
self.assertEqual(mock_mla.call_args.args[0].dtype, fp8_dtype)
|
||||
self.assertIs(mock_mla.call_args.kwargs["q_scale"], q_scale)
|
||||
mock_quant.assert_not_called()
|
||||
self.assertEqual(mock_mla.call_args.kwargs["q_scale"].item(), 1.0)
|
||||
self.assertTrue(mock_mla.call_args.kwargs["return_lse"])
|
||||
self.assertFalse(mock_mla.call_args.kwargs["intra_batch_mode"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user