diff --git a/python/sglang/kernels/ops/attention/dcp_kernels.py b/python/sglang/kernels/ops/attention/dcp_kernels.py index 30586e886e..82e23406db 100644 --- a/python/sglang/kernels/ops/attention/dcp_kernels.py +++ b/python/sglang/kernels/ops/attention/dcp_kernels.py @@ -125,6 +125,45 @@ def create_mla_kv_page_table_for_dcp( ) +@triton.jit +def compact_dcp_verify_token_table_to_ragged( + token_table_ptr, # in: [bs * q_len, table_stride] from create_mla_kv_page_table_for_dcp + local_kv_lens_ptr, # in: [bs] this rank's shard length per request, contiguous + kv_indptr_ptr, # in: [bs + 1] cumsum of the lengths, each clamped to >= 1 + kv_indices_ptr, # out: ragged ids, one run per request + table_stride: tl.constexpr, + Q_LEN: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + """Rectangular token table -> ragged (kv_indices, kv_indptr). + + Every row of a request repeats that request's shard, so this reads row 0. + """ + # The grid is (bs,) and the token loop is inside the kernel, so a cuda + # graph can capture this launch. + req = tl.program_id(0) + out_offset = tl.load(kv_indptr_ptr + req) + local_len = tl.load(local_kv_lens_ptr + req).to(tl.int32) + # The table must be wide enough for the whole shard. A narrow table makes + # this kernel read the next query row and return the wrong ids. + tl.device_assert(local_len <= table_stride) + + if local_len == 0: + # Give an empty shard one slot, because the attention kernel fails + # on a shard of length zero. The caller removes that row later. + tl.store(kv_indices_ptr + out_offset, 0) + else: + row = token_table_ptr + req * Q_LEN * table_stride + for i in range(tl.cdiv(local_len, BLOCK_SIZE)): + offset = tl.arange(0, BLOCK_SIZE).to(tl.int64) + i * BLOCK_SIZE + mask = offset < local_len + tl.store( + kv_indices_ptr + out_offset + offset, + tl.load(row + offset, mask=mask), + mask=mask, + ) + + @triton.jit def create_dcp_kv_indices( kv_indptr, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 62ca05fb7d..9b5f90d14b 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -193,6 +193,10 @@ class EnvIntWithAlias(_DeprecatedEnvFallback, EnvInt): pass +class EnvStrWithAlias(_DeprecatedEnvFallback, EnvStr): + pass + + class EnvFloat(EnvField): def parse(self, value: str) -> float: try: @@ -964,10 +968,14 @@ class Envs: # import and Triton cga_layout prerequisites hold. Set to 0 to force the # zero-pad mla_decode_fwd fallback (benchmarking / emergency disable). SGLANG_AITER_MLA_GLUON = EnvBool(True) - # Select the AITER MLA kernel for DCP decode. "gluon" preserves the native - # low-head Triton path; "asm" quantizes the DCP-gathered query to FP8 and - # runs mla_decode_fwd with persistent metadata and return_lse=True. - SGLANG_AITER_MLA_DCP_DECODE_BACKEND = EnvStr("gluon") + # Select the AITER MLA kernel for decode, "asm" or "gluon". + SGLANG_AITER_MLA_DECODE_BACKEND = EnvStrWithAlias( + "asm", deprecated_name="SGLANG_AITER_MLA_DCP_DECODE_BACKEND" + ) + # Select the AITER MLA kernel for target verify, "asm" or "gluon". + SGLANG_AITER_MLA_VERIFY_BACKEND = EnvStr("asm") + # Let aiter plan the KV splits for the asm persistent MLA decode. + SGLANG_AITER_MLA_AUTO_KV_SPLITS = EnvBool(False) # DSV4 Aiter flags SGLANG_OPT_USE_AITER_SILU_MUL = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 9fb035a114..a6da8049ab 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -72,7 +72,10 @@ except ImportError: "aiter is AMD specific kernel library. Please make sure aiter is installed on your AMD device." ) -from sglang.kernels.ops.attention.dcp_kernels import create_mla_kv_page_table_for_dcp +from sglang.kernels.ops.attention.dcp_kernels import ( + compact_dcp_verify_token_table_to_ragged, + create_mla_kv_page_table_for_dcp, +) from sglang.kernels.ops.attention.merge_state import merge_state_triton from sglang.kernels.ops.attention.utils import ( launch_reshape_and_cache_flash, @@ -104,6 +107,37 @@ _use_fp8_prefill_attn = ( get_bool_env_var("SGLANG_AITER_FP8_PREFILL_ATTN", "True") and is_gfx95_supported() ) +# Max qlen limitation of the asm ps kernels to handle q with bf16 dtype +_MLA_ASM_PS_MAX_QLEN = 4 + +# (query head count -> query lengths) at which aiter's scheduler treats the +# count as native, on gfx950 with an fp8 query and KV. None means any length. +# This map is copied from natively_supported in aiter/ops/attention.py. See +# https://github.com/ROCm/aiter/blob/bf37db00749722ae29ac27c101f534da43c586f5/aiter/ops/attention.py#L1669-L1706 +_MLA_ASM_CPRR_NATIVE_QLENS = { + 16: None, + 32: frozenset({4}), + 96: frozenset(range(1, 6 + 1)), + 128: None, +} + + +def _asm_cprr_supports_shape(heads: int, q_len: int) -> bool: + """Whether aiter schedules this head count natively at this query length.""" + if heads not in _MLA_ASM_CPRR_NATIVE_QLENS: + return False + allowed = _MLA_ASM_CPRR_NATIVE_QLENS[heads] + return allowed is None or q_len in allowed + + +def _asm_cprr_kernel_heads(gathered_heads: int, q_len: int) -> int: + """The smallest supported head count at or above the gathered one, or 0.""" + for heads in sorted(_MLA_ASM_CPRR_NATIVE_QLENS): + if heads >= gathered_heads and _asm_cprr_supports_shape(heads, q_len): + return heads + return 0 + + # (v_head_dim -> query head counts) that aiter's mla_reduce_v1 has an # instantiation. This map is copied from MLA_REDUCE_ROUTER in # aiter/csrc/kernels/mla/reduce.cu. See https://github.com/ROCm/aiter/blob/7915f53a4225b3f9cb632a97a23369dbebcf1be0/csrc/kernels/mla/reduce.cu#L923 @@ -186,6 +220,13 @@ class ForwardMetadata: prefill_ps_metadata: Optional[MlaPrefillPsMetadata] = None chunked_skip_prefix_ps_metadata: Optional[MlaPrefillPsMetadata] = None chunked_prefix_ps_metadatas: Optional[list[Optional[MlaPrefillPsMetadata]]] = None + # cprr + global_kv_indptr: Optional[torch.Tensor] = None + use_asm_cprr_verify: bool = False + cprr_kernel_heads: int = 0 + cprr_kv_indices: Optional[torch.Tensor] = None + cprr_kv_indptr: Optional[torch.Tensor] = None + cprr_empty_rows: Optional[torch.Tensor] = None def _build_paged_kv_view( @@ -313,6 +354,9 @@ class AiterAttnBackend(AttentionBackend): # can never carry more seqs than the pool. extend_dummy_seqs_capped_by_req_pool: bool = True + # declare here to avoid CI failure in test_aiter_fp8_q_unified_attention.py + use_mla_auto_kv_splits: bool = False + def __init__( self, model_runner: ModelRunner, @@ -350,18 +394,33 @@ class AiterAttnBackend(AttentionBackend): self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA self.dcp_world_size = get_parallel().attn_dcp_size - self.mla_dcp_decode_backend = ( - envs.SGLANG_AITER_MLA_DCP_DECODE_BACKEND.get().lower() - ) - if self.mla_dcp_decode_backend not in ("gluon", "asm"): + self.mla_decode_backend = envs.SGLANG_AITER_MLA_DECODE_BACKEND.get().lower() + if self.mla_decode_backend not in ("gluon", "asm"): raise ValueError( - "SGLANG_AITER_MLA_DCP_DECODE_BACKEND must be 'gluon' or 'asm', " - f"got {self.mla_dcp_decode_backend!r}" + "SGLANG_AITER_MLA_DECODE_BACKEND must be 'gluon' or 'asm', " + f"got {self.mla_decode_backend!r}" ) self.use_mla_dcp_asm = ( self.use_mla and self.dcp_world_size > 1 - and self.mla_dcp_decode_backend == "asm" + and self.mla_decode_backend == "asm" + ) + self.mla_verify_backend = envs.SGLANG_AITER_MLA_VERIFY_BACKEND.get().lower() + if self.mla_verify_backend not in ("gluon", "asm"): + raise ValueError( + "SGLANG_AITER_MLA_VERIFY_BACKEND must be 'gluon' or 'asm', " + f"got {self.mla_verify_backend!r}" + ) + self._cprr_launch_logged = False + # fast_mode / intra_batch_mode for the cp verify schedule. Not the + # module defaults: those size reduce_partial_map at 65536 entries + # instead of 1276, and mla_decode_fwd then asks for a 64 GiB logits + # buffer. + self.cprr_fast_mode = True + self.cprr_intra_batch_mode = False + gathered_heads = self.num_head * self.dcp_world_size + self.cprr_kernel_heads = _asm_cprr_kernel_heads( + gathered_heads, self.num_draft_tokens or 1 ) # Get v_head_dim based on model type @@ -570,10 +629,10 @@ class AiterAttnBackend(AttentionBackend): ) global _use_mla_ps_kernel, fast_mode, intra_batch_mode - # current mla_decode_fwd only support fake-nps in num_head == 16 - # so all num_head size does not use qh16 kernel to simulate - # it should not use fake-nps (fast_mode = False, intra_batch_mode = True) - # it will cause gpu-fault or accuracy issue. + # fake-nps (fast_mode False, intra_batch_mode True) picks aiter's + # v1_0 scheduler, which folds these counts to 16 while the gfx950 + # fp8 launch does not. Both sides must fold alike or the kernel + # faults. At 16 neither folds, so fake-nps is correct there. if self.mla_kernel_num_head_padded in (32, 64, 128): fast_mode = True intra_batch_mode = False @@ -592,14 +651,16 @@ class AiterAttnBackend(AttentionBackend): _use_mla_ps_kernel = False fast_mode = False intra_batch_mode = False - # Zero-pad topology (h12->qh16): prefer Gluon decode over PS kernel. if self.head_pad_mode == "zero" and self.kv_cache_dtype == fp8_dtype: # Disable ps only when gluon kernel is selected to avoid falling # back to incorrect aiter kernel - if prefer_mla_gluon_decode( - head_pad_mode=self.head_pad_mode, - num_head=self.num_head, - kv_cache_dtype=self.kv_cache_dtype, + if ( + prefer_mla_gluon_decode( + head_pad_mode=self.head_pad_mode, + num_head=self.num_head, + kv_cache_dtype=self.kv_cache_dtype, + ) + and not self._asm_ps_supports_decode_and_verify() ): _use_mla_ps_kernel = False fast_mode = False @@ -612,6 +673,27 @@ class AiterAttnBackend(AttentionBackend): if self.num_draft_tokens is None and use_any_mla_persist: self.max_split_per_batch = 64 + # When the env var is on, aiter plans the KV splits. The default + # scheduler gives every request ceil(num_cu / batch_size) splits. + # It ignores the KV length. That clamp is inside the kernel, so no + # host value lifts it. The v1_2 scheduler sizes the count from the + # workload. A max_split_per_batch of -1 selects its auto branch. + # + # Under DCP the cp verify route already runs the v1_2 scheduler: + # make_mla_decode_meta_data_buffer takes cprr_fast_mode and + # cprr_intra_batch_mode instead. There only the -1 has an effect. + self.use_mla_auto_kv_splits = ( + use_any_mla_persist and envs.SGLANG_AITER_MLA_AUTO_KV_SPLITS.get() + ) + if self.use_mla_auto_kv_splits: + logger.info( + "aiter MLA: aiter plans the KV splits " + "(SGLANG_AITER_MLA_AUTO_KV_SPLITS=1)" + ) + fast_mode = True + intra_batch_mode = False + self.max_split_per_batch = -1 + self.fix_max_split_per_batch = self.max_split_per_batch def pad_heads(self, x: torch.Tensor, padded: int) -> torch.Tensor: @@ -688,13 +770,21 @@ class AiterAttnBackend(AttentionBackend): *, metadata_fast_mode: Optional[bool] = None, metadata_intra_batch_mode: Optional[bool] = None, + nhead_override: Optional[int] = None, + max_split_per_batch: Optional[int] = None, ): + """Allocate the PS work-schedule buffers, sized for nhead_override.""" # Under DCP this is the gathered head count (num_head * dcp_world_size); # equals num_head_padded when DCP is off. - nhead = self.mla_kernel_num_head_padded + nhead = ( + self.mla_kernel_num_head_padded + if nhead_override is None + else nhead_override + ) dtype = self.kv_cache_dtype - if self.attn_dp_enabled: + # The auto count is -1, which this min() would turn into a real cap. + if self.attn_dp_enabled and not self.use_mla_auto_kv_splits: gpu = torch.cuda.current_device() device_properties = torch.cuda.get_device_properties(gpu) cu_num = device_properties.multi_processor_count @@ -728,6 +818,11 @@ class AiterAttnBackend(AttentionBackend): fast_mode=metadata_fast_mode, num_kv_splits=self.max_split_per_batch, intra_batch_mode=metadata_intra_batch_mode, + **( + {} + if max_split_per_batch is None + else {"max_split_per_batch": max_split_per_batch} + ), ) # aiter implementation @@ -777,16 +872,23 @@ class AiterAttnBackend(AttentionBackend): fast_mode, max_split_per_batch, intra_batch_mode, + is_cp_round_robin=False, + nhead_override=None, ): nhead_kv = 1 page_size = self.page_size dtype = self.kv_cache_dtype + nhead = ( + self.mla_kernel_num_head_padded + if nhead_override is None + else nhead_override + ) meta = get_mla_metadata_v1( qo_indptr, kv_indptr, kv_last_page_len, - self.mla_kernel_num_head_padded // nhead_kv, + nhead // nhead_kv, nhead_kv, False, work_metadata, @@ -801,6 +903,7 @@ class AiterAttnBackend(AttentionBackend): fast_mode=fast_mode, max_split_per_batch=max_split_per_batch, intra_batch_mode=intra_batch_mode, + is_cp_round_robin=is_cp_round_robin, dtype_q=dtype, dtype_kv=dtype, ) @@ -1287,6 +1390,52 @@ class AiterAttnBackend(AttentionBackend): f"Got topk={self.topk}." ) + def _asm_ps_supports_decode_and_verify(self) -> bool: + """Whether the asm PS MLA kernels support both decode and target verify.""" + verify_q_len = self.num_draft_tokens or 1 + if self.dcp_world_size > 1: + if _asm_cprr_supports_shape(self.cprr_kernel_heads, verify_q_len): + return True + logger.info( + "aiter MLA decode and verify stay on the Gluon kernel: the cp " + "verify kernel supports no %d-token window at %d gathered heads.", + verify_q_len, + self.num_head * self.dcp_world_size, + ) + return False + if self._asm_ps_supports_qlen(verify_q_len): + return True + logger.info( + "aiter MLA decode and verify stay on the Gluon kernel: target " + "verify is %d tokens wide and the asm kernels support at most %d at " + "%d heads. A DSPARK block size of %d or lower would fit.", + verify_q_len, + _MLA_ASM_PS_MAX_QLEN, + self.mla_kernel_num_head_padded, + _MLA_ASM_PS_MAX_QLEN - 1, + ) + return False + + def _asm_ps_supports_qlen(self, q_len: int) -> bool: + """Whether the asm PS dispatch has a kernel for this query length.""" + return q_len <= _MLA_ASM_PS_MAX_QLEN or ( + self.kv_cache_dtype == fp8_dtype + and self.mla_kernel_num_head_padded == 16 + and is_gfx95_supported() + ) + + def _asm_cprr_supports_verify_shape(self, q_len: Optional[int]) -> bool: + """Whether this topology has an asm cp kernel for DCP target verify.""" + return ( + _use_mla_ps_kernel + and self.mla_verify_backend == "asm" + and self.dcp_world_size > 1 + and self.kv_cache_dtype == fp8_dtype + and q_len is not None + and self.cprr_kernel_heads > 0 + and _asm_cprr_supports_shape(self.cprr_kernel_heads, q_len) + ) + def _mla_decode_fwd_with_head_pad( self, q: torch.Tensor, @@ -1301,6 +1450,27 @@ class AiterAttnBackend(AttentionBackend): q / o must already be shaped (..., num_head, head_dim). """ num_head = layer.tp_q_head_num + if ( + _use_mla_ps_kernel + and self.dcp_world_size <= 1 + and self._asm_ps_supports_qlen(kwargs.get("max_seqlen_q") or 1) + ): + # The asm kernel reads KV row 0 for masked positions and weights it + # by p = 0. A NaN there spreads to every request, because + # 0 * NaN = NaN. Row 0 is where CUDA-graph padding tokens write + # their KV, and that KV can be NaN. Gluon, the kernel this path + # replaces, does not read row 0. + k_buffer_flat[0].zero_() + if ( + self.kv_cache_dtype == fp8_dtype + and self.mla_kernel_num_head_padded == 16 + and is_gfx95_supported() + and (kwargs.get("max_seqlen_q") or 1) > _MLA_ASM_PS_MAX_QLEN + and q.dtype != fp8_dtype + ): + q_flat = q.reshape(q.shape[0], -1).to(fp8_dtype) + kwargs["q_scale"] = torch.ones((), dtype=torch.float32, device=q.device) + q = q_flat.view(q.shape) if self.head_pad_mode == "repeat" or ( self.head_pad_mode == "none" and self.num_head_padded != self.num_head ): @@ -1367,6 +1537,17 @@ class AiterAttnBackend(AttentionBackend): return 1 return int(seq_lens.max().item()) + def _kernel_num_kv_splits(self) -> Optional[int]: + """The split count for a decode kernel call, None to let aiter pick. + + The metadata call takes an int and gets -1 for the auto branch. The + kernel takes None instead. mla.py re-decides persistent mode only on + None. It would read -1 as a real count. + """ + if self.use_mla_auto_kv_splits: + return None + return self.forward_metadata.num_kv_splits + def _forward_mla_decode( self, q: torch.Tensor, @@ -1382,6 +1563,10 @@ class AiterAttnBackend(AttentionBackend): head_pad_mode=getattr(self, "head_pad_mode", "none"), num_head=getattr(self, "num_head", layer.tp_q_head_num), kv_cache_dtype=self.kv_cache_dtype, + ) and not ( + _use_mla_ps_kernel + and self.mla_decode_backend == "asm" + and self._asm_ps_supports_qlen(max_q_len) ): return mla_gluon_decode( q=q, @@ -1401,7 +1586,7 @@ class AiterAttnBackend(AttentionBackend): reduce_indptr = self.forward_metadata.reduce_indptr reduce_final_map = self.forward_metadata.reduce_final_map reduce_partial_map = self.forward_metadata.reduce_partial_map - num_kv_splits = self.forward_metadata.num_kv_splits + num_kv_splits = self._kernel_num_kv_splits() return self._mla_decode_fwd_with_head_pad( q, @@ -1426,10 +1611,17 @@ class AiterAttnBackend(AttentionBackend): num_kv_splits=num_kv_splits, ) - def _get_dcp_graph_max_local_kv_len(self) -> int: - """Static upper bound on this rank's shard, ceil(max_context_len / W).""" + def _dcp_max_local_kv_len(self, max_global_kv_len: int) -> int: + """Widest shard one rank can hold for a global kv length.""" w = max(self.dcp_world_size, 1) - return (self.max_context_len + w - 1) // w + return (max_global_kv_len + w - 1) // w + + def _dcp_graph_max_local_kv_len(self) -> int: + """Static upper bound on this rank's shard under a cuda graph.""" + max_global_kv_len = self.max_context_len + if self._asm_cprr_supports_verify_shape(self.num_draft_tokens): + max_global_kv_len += self.num_draft_tokens + return self._dcp_max_local_kv_len(max_global_kv_len) def _forward_decode_dcp(self, q, k_buffer, layer, k_descale): """Attend this rank's KV shard for decode -> (out, natural-log lse).""" @@ -1446,8 +1638,12 @@ class AiterAttnBackend(AttentionBackend): q_mla = q.view(bs, num_heads, layer.qk_head_dim) q_scale = k_descale if q_mla.dtype != fp8_dtype: - q_mla, q_scale = scaled_fp8_quant(q_mla.reshape(bs, -1)) - q_mla = q_mla.view(bs, num_heads, layer.qk_head_dim) + q_mla = ( + q_mla.reshape(bs, -1) + .to(fp8_dtype) + .view(bs, num_heads, layer.qk_head_dim) + ) + q_scale = torch.ones((), dtype=torch.float32, device=q.device) out = torch.empty( (bs, num_heads, layer.v_head_dim), @@ -1465,7 +1661,7 @@ class AiterAttnBackend(AttentionBackend): fm.max_q_len or 1, sm_scale=layer.scaling, logit_cap=layer.logit_cap, - num_kv_splits=fm.num_kv_splits, + num_kv_splits=self._kernel_num_kv_splits(), work_meta_data=fm.work_metadata, work_indptr=fm.work_indptr, work_info_set=fm.work_info_set, @@ -1496,13 +1692,126 @@ class AiterAttnBackend(AttentionBackend): ) return out, lse.view(bs, num_heads) - def _forward_verify_dcp(self, q, k_window, layer, k_descale): - """Attend the committed prefix and the verify window separately, then - merge -> (out, natural-log lse). + def _forward_verify_asm_cprr(self, q, layer, k_descale, n_rows): + """DCP target verify on the asm cp round-robin kernel: one masked pass.""" + fm = self.forward_metadata + num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp + kernel_heads = fm.cprr_kernel_heads or num_heads + if not self._cprr_launch_logged: + self._cprr_launch_logged = True + logger.info( + "aiter DCP cp verify: gathered=%d kernel_heads=%d%s " + "intra_batch=%s max_q_len=%d reduce_partial_map=%d", + num_heads, + kernel_heads, + " (padded)" if kernel_heads != num_heads else " (native)", + self.cprr_intra_batch_mode, + fm.max_q_len, + fm.reduce_partial_map.numel(), + ) + # torch.full, not torch.as_tensor: as_tensor copies from host memory, + # which a cuda graph capture cannot record. + kv_scale = torch.full( + (), + self._resolve_fp8_kv_scale_float(layer, k_descale), + dtype=torch.float32, + device=q.device, + ) + kv_indices, kv_indptr = fm.cprr_kv_indices, fm.cprr_kv_indptr + q_in = ( + q.view(n_rows, -1).to(fp8_dtype).view(n_rows, num_heads, layer.qk_head_dim) + ) + q_scale = torch.ones((), dtype=torch.float32, device=q.device) + if kernel_heads != num_heads: + q_in = self.pad_heads(q_in, kernel_heads) + out = q.new_empty( + (n_rows, kernel_heads, layer.v_head_dim), dtype=self.input_dtype + ) + _, lse = mla_decode_fwd( + q_in, + self.token_to_kv_pool.get_key_buffer(layer.layer_id).view( + -1, 1, 1, layer.qk_head_dim + ), + out, + fm.qo_indptr, + kv_indptr, + kv_indices, + fm.kv_last_page_len, + fm.max_q_len, + sm_scale=layer.scaling, + work_meta_data=fm.work_metadata, + work_indptr=fm.work_indptr, + work_info_set=fm.work_info_set, + reduce_indptr=fm.reduce_indptr, + reduce_final_map=fm.reduce_final_map, + reduce_partial_map=fm.reduce_partial_map, + q_scale=q_scale, + kv_scale=kv_scale, + intra_batch_mode=self.cprr_intra_batch_mode, + num_kv_splits=self._kernel_num_kv_splits(), + return_lse=True, + g_kv_indptr=fm.global_kv_indptr, + cp_world_size=self.dcp_world_size, + cp_rank=get_parallel().attn_dcp_rank, + causal=True, + ) + lse = lse.view(n_rows, kernel_heads) + if kernel_heads != num_heads: + out = out[:, :num_heads, :] + lse = lse[:, :num_heads] + return out, lse - Splitting at the window boundary avoids the one thing the decode kernel - cannot do under DCP: mask on the GLOBAL position g(j) = j * W + r. - """ + def _build_dcp_verify_ragged_indices( + self, + verify_token_table: torch.Tensor, + local_kv_lens: torch.Tensor, + bs: int, + q_len: int, + out_indices: Optional[torch.Tensor] = None, + out_indptr: Optional[torch.Tensor] = None, + out_empty: Optional[torch.Tensor] = None, + ): + """This rank's verify shard as ragged (kv_indices, kv_indptr, empty_rows).""" + # Pass out_* to use the address-stable buffers of the cuda graph. + # Omit them to allocate new tensors. + # The .contiguous() call is necessary. The slice has stride q_len, and + # .to() returns an int32 tensor unchanged, so without the copy the + # kernel reads lens[req] instead of lens[req * q_len]. The two agree + # only at batch size 1. + lens = local_kv_lens[: bs * q_len : q_len].to(torch.int32).contiguous() + # The kernel faults on a zero-length shard, so give every shard a slot. + clamped = lens.clamp_min(1) + if out_indptr is None: + kv_indptr = torch.zeros(bs + 1, dtype=torch.int32, device=lens.device) + else: + kv_indptr = out_indptr[: bs + 1] + # Not kv_indptr[0] = 0: that copies from host memory, which a + # cuda graph capture cannot record. + kv_indptr[:1].zero_() + torch.cumsum(clamped, dim=0, out=kv_indptr[1:]) + + if out_indices is None: + kv_indices = verify_token_table.new_empty( + (int(clamped.sum().item()),), dtype=torch.int32 + ) + else: + kv_indices = out_indices + compact_dcp_verify_token_table_to_ragged[(bs,)]( + verify_token_table, + lens, + kv_indptr, + kv_indices, + table_stride=verify_token_table.shape[1], + Q_LEN=q_len, + BLOCK_SIZE=_DCP_VERIFY_TABLE_COLS_PER_BLOCK, + ) + empty_rows = local_kv_lens[: bs * q_len].eq(0) + if out_empty is not None: + empty_rows = out_empty[: bs * q_len].copy_(empty_rows) + return kv_indices, kv_indptr, empty_rows + + def _forward_verify_dcp(self, q, k_window, layer, k_descale): + """DCP target verify -> (out, natural-log lse) for the cross-rank merge.""" fm = self.forward_metadata q_len = fm.max_q_len num_heads = layer.tp_q_head_num # gathered heads = num_local_heads * dcp @@ -1510,6 +1819,13 @@ class AiterAttnBackend(AttentionBackend): n_rows = seqused_k.shape[0] bs = n_rows // q_len + if fm.use_asm_cprr_verify: + out_a, lse_a = self._forward_verify_asm_cprr(q, layer, k_descale, n_rows) + empty = fm.cprr_empty_rows + out_a.masked_fill_(empty[:, None, None], 0.0) + lse_a.masked_fill_(empty[:, None], -1e30) + return out_a, lse_a + out_a, lse_a = mla_gluon_decode( q=q.view(n_rows, num_heads, layer.qk_head_dim), k_buffer=self.token_to_kv_pool.get_key_buffer(layer.layer_id), @@ -1730,6 +2046,9 @@ class AiterAttnBackend(AttentionBackend): # dcp metadata local_kv_lens = None verify_token_table = None + global_kv_indptr = None + use_asm_cprr_verify = False + cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None if forward_batch.forward_mode.is_decode_or_idle(): if spec_info is None or forward_batch.forward_mode.is_idle(): kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) @@ -1967,7 +2286,8 @@ class AiterAttnBackend(AttentionBackend): if self.use_mla: draft_num = spec_info.draft_token_num device = forward_batch.seq_lens.device - if self.dcp_world_size > 1: + use_asm_cprr_verify = self._asm_cprr_supports_verify_shape(draft_num) + if self.dcp_world_size > 1 and not use_asm_cprr_verify: kv_lens = forward_batch.seq_lens.to(torch.int32).clone() kv_lens_sum = forward_batch.seq_lens_sum else: @@ -2001,6 +2321,7 @@ class AiterAttnBackend(AttentionBackend): ) if self.dcp_world_size > 1: + global_kv_indptr = kv_indptr.clone() self._plan_dcp_decode_metadata( kv_indptr, kv_indices, @@ -2016,11 +2337,27 @@ class AiterAttnBackend(AttentionBackend): forward_batch.req_pool_indices, bs, draft_num, - (max_kv_len + self.dcp_world_size - 1) // self.dcp_world_size, + self._dcp_max_local_kv_len( + max_kv_len + (draft_num if use_asm_cprr_verify else 0) + ), ) - if _use_mla_ps_kernel and self.dcp_world_size <= 1: + if use_asm_cprr_verify: + ( + cprr_kv_indices, + cprr_kv_indptr, + cprr_empty_rows, + ) = self._build_dcp_verify_ragged_indices( + verify_token_table, local_kv_lens, bs, draft_num + ) + if _use_mla_ps_kernel and ( + self.dcp_world_size <= 1 or use_asm_cprr_verify + ): max_seqlen_qo = draft_num + is_cp_round_robin = self.dcp_world_size > 1 + metadata_heads = ( + self.cprr_kernel_heads if use_asm_cprr_verify else None + ) ( work_metadata, work_indptr, @@ -2028,13 +2365,26 @@ class AiterAttnBackend(AttentionBackend): reduce_indptr, reduce_final_map, reduce_partial_map, - ) = self.make_mla_decode_meta_data_buffer(max_seqlen_qo, bs) + ) = self.make_mla_decode_meta_data_buffer( + max_seqlen_qo, + bs, + metadata_fast_mode=( + self.cprr_fast_mode if use_asm_cprr_verify else None + ), + metadata_intra_batch_mode=( + self.cprr_intra_batch_mode if use_asm_cprr_verify else None + ), + nhead_override=metadata_heads, + max_split_per_batch=( + self.max_split_per_batch if use_asm_cprr_verify else None + ), + ) num_kv_splits = self.max_split_per_batch self.make_mla_meta_data( qo_indptr, - kv_indptr, + cprr_kv_indptr if use_asm_cprr_verify else kv_indptr, self.kv_last_page_len[:bs], work_metadata, work_info_set, @@ -2043,9 +2393,17 @@ class AiterAttnBackend(AttentionBackend): reduce_final_map, reduce_partial_map, max_seqlen_qo, - fast_mode=fast_mode, + fast_mode=( + self.cprr_fast_mode if use_asm_cprr_verify else fast_mode + ), max_split_per_batch=num_kv_splits, - intra_batch_mode=intra_batch_mode, + intra_batch_mode=( + self.cprr_intra_batch_mode + if use_asm_cprr_verify + else intra_batch_mode + ), + is_cp_round_robin=is_cp_round_robin, + nhead_override=metadata_heads, ) self.forward_metadata = ForwardMetadata( @@ -2066,6 +2424,14 @@ class AiterAttnBackend(AttentionBackend): run_graph=False, local_kv_lens=local_kv_lens, verify_token_table=verify_token_table, + global_kv_indptr=global_kv_indptr, + use_asm_cprr_verify=use_asm_cprr_verify, + cprr_kernel_heads=self.cprr_kernel_heads + if use_asm_cprr_verify + else 0, + cprr_kv_indices=cprr_kv_indices, + cprr_kv_indptr=cprr_kv_indptr, + cprr_empty_rows=cprr_empty_rows, ) else: draft_num = forward_batch.input_ids.shape[0] // bs @@ -2395,10 +2761,8 @@ class AiterAttnBackend(AttentionBackend): self.cuda_graph_verify_local_kv_lens = torch.zeros( (n_verify_rows,), dtype=torch.int32, device=self.device ) - # Capture-stable token table, filled out-of-graph. One column per - # TOKEN, so the width is the worst-case shard ceil(ctx_len / W). self.cuda_graph_verify_token_table = torch.zeros( - (n_verify_rows, self._get_dcp_graph_max_local_kv_len()), + (n_verify_rows, self._dcp_graph_max_local_kv_len()), dtype=torch.int32, device=self.device, ) @@ -2406,10 +2770,24 @@ class AiterAttnBackend(AttentionBackend): # verify plan without a sync; see _plan_dcp_decode_metadata. self.cuda_graph_dcp_static_local_kv_lens = torch.full( (max_bs,), - self._get_dcp_graph_max_local_kv_len(), + self._dcp_graph_max_local_kv_len(), dtype=torch.int32, device="cpu", ) + self.cuda_graph_global_kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=self.device + ) + self.cuda_graph_cprr_kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=self.device + ) + self.cuda_graph_cprr_kv_indices = torch.zeros( + (max_bs * (self._dcp_graph_max_local_kv_len() + 1),), + dtype=torch.int32, + device=self.device, + ) + self.cuda_graph_cprr_empty_rows = torch.zeros( + (n_verify_rows,), dtype=torch.bool, device=self.device + ) if kv_indices_buf is None: max_num_blocks_per_seq = ( self.max_context_len + self.page_size - 1 @@ -2471,6 +2849,9 @@ class AiterAttnBackend(AttentionBackend): (True, False) if self.use_mla_dcp_asm else (fast_mode, intra_batch_mode) ) + graph_use_asm_cprr_verify = self._asm_cprr_supports_verify_shape( + self.num_draft_tokens + ) ( self.work_metadata, self.work_indptr, @@ -2481,8 +2862,37 @@ class AiterAttnBackend(AttentionBackend): ) = self.make_mla_decode_meta_data_buffer( max_seqlen_qo, max_bs, - metadata_fast_mode=metadata_fast_mode, - metadata_intra_batch_mode=metadata_intra_batch_mode, + metadata_fast_mode=( + self.cprr_fast_mode + if graph_use_asm_cprr_verify + else metadata_fast_mode + ), + metadata_intra_batch_mode=( + self.cprr_intra_batch_mode + if graph_use_asm_cprr_verify + else metadata_intra_batch_mode + ), + nhead_override=( + self.cprr_kernel_heads if graph_use_asm_cprr_verify else None + ), + max_split_per_batch=( + self.max_split_per_batch if graph_use_asm_cprr_verify else None + ), + ) + logger.info( + "aiter DCP cp verify graph buffers: use_asm_cprr_verify=%s num_draft_tokens=%s " + "max_seqlen_qo=%d max_bs=%d nhead=%s intra=%s split_cap=%s " + "reduce_partial_map=%d", + graph_use_asm_cprr_verify, + self.num_draft_tokens, + max_seqlen_qo, + max_bs, + self.cprr_kernel_heads if graph_use_asm_cprr_verify else None, + self.cprr_intra_batch_mode + if graph_use_asm_cprr_verify + else metadata_intra_batch_mode, + self.max_split_per_batch if graph_use_asm_cprr_verify else None, + self.reduce_partial_map.numel(), ) else: @@ -2536,6 +2946,9 @@ class AiterAttnBackend(AttentionBackend): # DCP metadata which will be populated for MLA decode when dcp enabled local_kv_lens = None verify_token_table = None + global_kv_indptr = None + cprr_kv_indices = cprr_kv_indptr = cprr_empty_rows = None + use_asm_cprr_verify = False swa_page_table = None # Unified verify never reads this host max (replay sizes from @@ -2712,9 +3125,12 @@ class AiterAttnBackend(AttentionBackend): device=self.device, ) if self.use_mla: + use_asm_cprr_verify = self._asm_cprr_supports_verify_shape( + self.num_draft_tokens + ) kv_lens = ( seq_lens - if self.dcp_world_size > 1 + if (self.dcp_world_size > 1 and not use_asm_cprr_verify) else seq_lens + self.num_draft_tokens ) else: @@ -2748,6 +3164,8 @@ class AiterAttnBackend(AttentionBackend): kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs] if self.use_mla and self.dcp_world_size > 1: + global_kv_indptr = self.cuda_graph_global_kv_indptr[: bs + 1] + global_kv_indptr.copy_(kv_indptr[: bs + 1]) self._plan_dcp_decode_metadata( kv_indptr, kv_indices, @@ -2767,19 +3185,36 @@ class AiterAttnBackend(AttentionBackend): req_pool_indices, bs, self.num_draft_tokens, - self._get_dcp_graph_max_local_kv_len(), + self._dcp_graph_max_local_kv_len(), out=self.cuda_graph_verify_token_table[:n_rows], out_lens=self.cuda_graph_verify_local_kv_lens[:n_rows], ) + if use_asm_cprr_verify: + ( + cprr_kv_indices, + cprr_kv_indptr, + cprr_empty_rows, + ) = self._build_dcp_verify_ragged_indices( + verify_token_table, + local_kv_lens, + bs, + self.num_draft_tokens, + out_indices=self.cuda_graph_cprr_kv_indices, + out_indptr=self.cuda_graph_cprr_kv_indptr, + out_empty=self.cuda_graph_cprr_empty_rows, + ) if self.use_mla: max_q_len = self.num_draft_tokens - if _use_mla_ps_kernel and self.dcp_world_size <= 1: + if _use_mla_ps_kernel and ( + self.dcp_world_size <= 1 or use_asm_cprr_verify + ): num_kv_splits = self.max_split_per_batch + is_cp_round_robin = self.dcp_world_size > 1 self.make_mla_meta_data( qo_indptr, - kv_indptr, + cprr_kv_indptr if use_asm_cprr_verify else kv_indptr, kv_last_page_len, self.work_metadata, self.work_info_set, @@ -2788,9 +3223,19 @@ class AiterAttnBackend(AttentionBackend): self.reduce_final_map, self.reduce_partial_map, max_q_len, - fast_mode=fast_mode, + fast_mode=( + self.cprr_fast_mode if use_asm_cprr_verify else fast_mode + ), max_split_per_batch=num_kv_splits, - intra_batch_mode=intra_batch_mode, + intra_batch_mode=( + self.cprr_intra_batch_mode + if use_asm_cprr_verify + else intra_batch_mode + ), + is_cp_round_robin=is_cp_round_robin, + nhead_override=( + self.cprr_kernel_heads if use_asm_cprr_verify else None + ), ) work_metadata = self.work_metadata @@ -2817,6 +3262,14 @@ class AiterAttnBackend(AttentionBackend): num_kv_splits=num_kv_splits, local_kv_lens=local_kv_lens, verify_token_table=verify_token_table, + global_kv_indptr=global_kv_indptr, + use_asm_cprr_verify=use_asm_cprr_verify, + cprr_kernel_heads=self.cprr_kernel_heads + if use_asm_cprr_verify + else 0, + cprr_kv_indices=cprr_kv_indices, + cprr_kv_indptr=cprr_kv_indptr, + cprr_empty_rows=cprr_empty_rows, ) else: max_q_len = verify_tokens_per_req @@ -3368,8 +3821,6 @@ class AiterAttnBackend(AttentionBackend): forward_batch.forward_mode.is_target_verify() and self.dcp_world_size > 1 ): - # two-stage dcp verify, dispatched before the dims below: the - # model provides the per rank kvcache slices. return self._forward_verify_dcp(q, k, layer, k_descale) qk_nope_head_dim = k.shape[-1] - qk_rope_head_dim @@ -3534,6 +3985,10 @@ class AiterAttnBackend(AttentionBackend): head_pad_mode=getattr(self, "head_pad_mode", "none"), num_head=getattr(self, "num_head", layer.tp_q_head_num), kv_cache_dtype=self.kv_cache_dtype, + ) and not ( + _use_mla_ps_kernel + and self.mla_verify_backend == "asm" + and self._asm_ps_supports_qlen(self.forward_metadata.max_q_len or 1) ): return mla_gluon_decode( q=q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), diff --git a/test/registered/attention/test_aiter_gluon_h12_fp8.py b/test/registered/attention/test_aiter_gluon_h12_fp8.py index b3153021ea..d20f4cc8b8 100644 --- a/test/registered/attention/test_aiter_gluon_h12_fp8.py +++ b/test/registered/attention/test_aiter_gluon_h12_fp8.py @@ -237,6 +237,10 @@ class TestForwardMlaDecodeDispatch(CustomTestCase): be.kv_cache_dtype = fp8_dtype be.head_pad_mode = "zero" be.num_head_padded = 16 + # The head count the asm kernel runs at. Without DCP it is the padded + # per-rank count, so 16 here. _asm_ps_supports_qlen reads it. + be.mla_kernel_num_head_padded = 16 + be.mla_decode_backend = "asm" be.forward_metadata = mock.Mock( max_q_len=max_q_len, kv_indices=torch.zeros(4, dtype=torch.int32), @@ -283,7 +287,9 @@ class TestForwardMlaDecodeDispatch(CustomTestCase): return_value=True, ) @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode") + @mock.patch("sglang.srt.layers.attention.aiter_backend._use_mla_ps_kernel", False) def test_uses_gluon_output_when_preferred(self, mock_gluon, _prefer): + """Gluon wins when it is preferred and the asm PS kernel is off.""" gluon_out = torch.ones(4, 12, 512) mock_gluon.return_value = gluon_out be = self._make_backend() @@ -302,9 +308,52 @@ class TestForwardMlaDecodeDispatch(CustomTestCase): return_value=True, ) @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode") + def test_asm_ps_wins_over_preferred_gluon(self, mock_gluon, _prefer): + """The asm PS kernel takes a width it supports, even over preferred Gluon.""" + be = self._make_backend() + be._forward_mla_decode( + torch.zeros(4, 12, 576, dtype=torch.bfloat16), + _layer(), + mock.Mock(), + k_descale=1.0, + ) + mock_gluon.assert_not_called() + be._mla_decode_fwd_with_head_pad.assert_called_once() + + @mock.patch( + "sglang.srt.layers.attention.aiter_backend.prefer_mla_gluon_decode", + return_value=True, + ) + @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode") + def test_decode_backend_gluon_keeps_gluon(self, mock_gluon, _prefer): + """SGLANG_AITER_MLA_DECODE_BACKEND=gluon holds decode on Gluon.""" + gluon_out = torch.ones(4, 12, 512) + mock_gluon.return_value = gluon_out + be = self._make_backend() + be.mla_decode_backend = "gluon" + out = be._forward_mla_decode( + torch.zeros(4, 12, 576, dtype=torch.bfloat16), + _layer(), + mock.Mock(), + k_descale=1.0, + ) + mock_gluon.assert_called_once() + be._mla_decode_fwd_with_head_pad.assert_not_called() + self.assertIs(out, gluon_out) + + @mock.patch( + "sglang.srt.layers.attention.aiter_backend.prefer_mla_gluon_decode", + return_value=True, + ) + @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_gluon_decode") + @mock.patch("sglang.srt.layers.attention.aiter_backend._use_mla_ps_kernel", False) def test_passes_max_q_len_as_qlen(self, mock_gluon, _prefer): - """Target-verify must reach the kernel as qlen, not be silently dropped: - the ASM fallback cannot serve this topology above qSeqLen 4 at all.""" + """Target verify must reach the kernel as qlen, not be silently dropped. + + The asm PS kernel is off here, so the width is Gluon's to serve. With + it on, a bf16 query stops at qSeqLen 4 and an fp8 query goes above it; + _asm_ps_supports_qlen owns that decision. + """ mock_gluon.return_value = torch.ones(32, 12, 512) be = self._make_backend(max_q_len=8) be._forward_mla_decode( @@ -353,16 +402,18 @@ class TestMlaDcpAsmDecode(CustomTestCase): @mock.patch("sglang.srt.layers.attention.aiter_backend.mla_decode_fwd", create=True) @mock.patch("sglang.srt.layers.attention.aiter_backend.scaled_fp8_quant") - def test_asm_quantizes_gathered_q_and_returns_lse(self, mock_quant, mock_mla): + def test_asm_casts_gathered_q_and_returns_lse(self, mock_quant, mock_mla): + """Q reaches the kernel as a plain fp8 cast at a scale of 1.0. + + Scaling by the tensor maximum first is no more accurate on this query + and costs about twice the time, so scaled_fp8_quant is not called. + """ from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype be = self._make_backend() layer = self._make_layer() q = torch.zeros(2, 96, 576, dtype=torch.bfloat16) - q_fp8 = torch.zeros(2, 96 * 576, dtype=fp8_dtype) - q_scale = torch.tensor([0.25], dtype=torch.float32) lse = torch.arange(2 * 96, dtype=torch.float32).view(2, 96) - mock_quant.return_value = (q_fp8, q_scale) mock_mla.return_value = (None, lse) out, actual_lse = be._forward_decode_dcp( @@ -376,7 +427,8 @@ class TestMlaDcpAsmDecode(CustomTestCase): self.assertEqual(out.dtype, torch.bfloat16) self.assertTrue(torch.equal(actual_lse, lse)) self.assertEqual(mock_mla.call_args.args[0].dtype, fp8_dtype) - self.assertIs(mock_mla.call_args.kwargs["q_scale"], q_scale) + mock_quant.assert_not_called() + self.assertEqual(mock_mla.call_args.kwargs["q_scale"].item(), 1.0) self.assertTrue(mock_mla.call_args.kwargs["return_lse"]) self.assertFalse(mock_mla.call_args.kwargs["intra_batch_mode"])