diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx index 26b1a53dcc..1e0f0496f9 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1.mdx @@ -201,6 +201,6 @@ To enable HiCache, open the **HiCache** card in the [Playground above](#playgrou The Write policy knob controls the GPU → CPU write and defaults to `write_through` (the upstream default): every page is mirrored to the CPU tier as it is written. `write_through_selective` backs up only hot data and `write_back` defers the copy to eviction, trading cache freshness for host-side I/O. -The card is not offered on MI350X: the ROCm recipes run `--disable-radix-cache`, and the server rejects that alongside `--enable-hierarchical-cache`. +The card is not offered on MI350X: hierarchical cache has not been validated on ROCm for this model yet. Only the L2 tier is exposed here. For the storage (L3) tier and the canonical flag set, see the [HiCache best-practices recipe](../../../docs/advanced_features/hicache_best_practices) and the [HiCache documentation](../../../docs/advanced_features/hicache). diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 24ca0f0861..ac93326d32 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1882,6 +1882,11 @@ SGLang supports various environment variables that can be used to configure its Disable linear-layer quantization on ROCm. false + + SGLANG_HACK_FLASHMLA_BACKEND + DeepSeek-V4 HIP decode attention kernel: auto takes aiter's gluon sparse kernel on gfx950 and the tilelang partial + combine elsewhere; aiter_sparse, tilelang, triton, torch, comparison or unified_kv_triton force one. + auto + SGLANG_ROCM_K3_FUSE_KDA_INPROJ Kimi-K3 on ROCm: fold the KDA [f_a|b] tail into the wide [q,k,v,g] projection so the whole input projection is one GEMM. Applies to unquantized weights only; falls back to the split projection otherwise. diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4_1.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4_1.jsx index b1adb3d41a..6f559123cf 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4_1.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4_1.jsx @@ -4,10 +4,9 @@ // real weights. // // Every DSpark cell caps --cuda-graph-max-bs-decode: the derived batch list does -// not fit while capturing the DSpark decode graphs, on any NVIDIA platform. The -// MI350X cell already carried the equivalent --cuda-graph-max-bs. H200 is the +// not fit while capturing the DSpark decode graphs, on any platform. H200 is the // tightest board at 140 GiB and needs the cap on both cells plus a lower memory -// fraction; the three other High-Throughput cells start without either. +// fraction; the B200 / B300 / GB300 High-Throughput cells start without either. // // DP-Attention, DeepEP and MegaMoE are absent by design: they have never been // enabled on this model. EP is set equal to TP on every shape here. @@ -122,9 +121,8 @@ export const config = { }, }, - // GPU → CPU KV offload (L2 only; no storage tier). Hidden on MI350X: both - // ROCm cells run `--disable-radix-cache`, which the server rejects alongside - // `--enable-hierarchical-cache`. + // GPU → CPU KV offload (L2 only; no storage tier). Hidden on MI350X: not + // validated on ROCm for this model yet. hicache: { excludesHw: ["mi350x"], writePolicies: [ @@ -343,35 +341,36 @@ export const config = { ], }, - // ---------- MI350X: 4x MI350X (gfx950), TP4 + EP4. Speculative decoding is - // rejected on ROCm, so there is one recipe. ---------- + // ---------- MI350X: 4x MI350X (gfx950), TP4 + EP4. The backends resolve + // automatically on HIP, and the radix cache works (turning it off cost ~10% at + // 32 concurrent requests). Both cells keep DSpark: on MI350X it beat plain decode + // at every load measured, up to 256 concurrent requests. Decoder SWA bounded + // replay cuts prefill work at no decode cost (TTFT -33% at 128K input with a 90% + // prefix hit, and +17-23% throughput at 64-256 requests). ---------- { - // DSpark runs on MI350X but is off by default; this cell turns it on. match: { hw: "mi350x", strategy: "low-latency" }, nnodes: 1, verified: true, env: [ - // Load-bearing: without it the fp4 experts land in the Triton - // fused-experts runner and assert on the hidden size. - "SGLANG_USE_AITER=1", - "SGLANG_MOE_PADDING=1", - // Required for run-to-run repeatable output: forces the FlyDSL MoE - // down-projection onto a per-slot reduce instead of atomics. + // Run-to-run repeatable output: forces the FlyDSL MoE down-projection onto + // a per-slot reduce instead of atomics; measured free at 8-32 requests. "AITER_FLYDSL_FORCE_REDUCE=1", - "ROCM_QUICK_REDUCE_QUANTIZATION=NONE", + // Required until ROCm/aiter#5802 is in the aiter pin: below 256 tokens aiter + // routes these fp4 experts to a bf16-activation MoE path the pin cannot run. + "AITER_BF16_FP8_MOE_BOUND=0", ], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--disable-radix-cache", "--mem-fraction-static 0.8", "--speculative-algorithm DSPARK", "--speculative-dspark-block-size 5", - "--cuda-graph-max-bs-decode 64", - "--cuda-graph-backend-prefill breakable", - "--cuda-graph-max-bs-prefill 4096", + // Decode CUDA graphs: the derived batch list does not fit here; a cap of 128 + // fits and keeps 128-request decode batches on the graph. + "--cuda-graph-max-bs-decode 128", + "--enable-decoder-swa-bounded-replay", "--reasoning-parser auto", "--tool-call-parser auto", "--host {{HOST_IP}}", @@ -379,29 +378,26 @@ export const config = { ], }, { - // The attention backend and mem-fraction-static are the resolved - // defaults on HIP, so this cell leaves both alone. match: { hw: "mi350x", strategy: "high-throughput" }, nnodes: 1, verified: true, env: [ - // Load-bearing: without it the fp4 experts land in the Triton - // fused-experts runner and assert on the hidden size. - "SGLANG_USE_AITER=1", - "SGLANG_MOE_PADDING=1", - // Required for run-to-run repeatable output: forces the FlyDSL MoE - // down-projection onto a per-slot reduce instead of atomics. "AITER_FLYDSL_FORCE_REDUCE=1", - "ROCM_QUICK_REDUCE_QUANTIZATION=NONE", + "AITER_BF16_FP8_MOE_BOUND=0", ], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--tp 4", "--ep-size 4", - "--disable-radix-cache", - "--cuda-graph-backend-prefill breakable", - "--cuda-graph-max-bs-prefill 4096", + "--mem-fraction-static 0.8", + // DSpark with graphs up to the running-request cap: 47% more throughput + // than plain decode at 256 concurrent requests. + "--speculative-algorithm DSPARK", + "--speculative-dspark-block-size 5", + "--cuda-graph-max-bs-decode 256", + "--max-running-requests 256", + "--enable-decoder-swa-bounded-replay", "--reasoning-parser auto", "--tool-call-parser auto", "--host {{HOST_IP}}", diff --git a/python/sglang/kernels/ops/layernorm/mhc_post_split_h.py b/python/sglang/kernels/ops/layernorm/mhc_post_split_h.py index 2331fe18c8..a36835f2fe 100644 --- a/python/sglang/kernels/ops/layernorm/mhc_post_split_h.py +++ b/python/sglang/kernels/ops/layernorm/mhc_post_split_h.py @@ -26,7 +26,12 @@ def _mhc_post_split_h_kernel(X, R, P, C, Y, H: tl.constexpr, B: tl.constexpr): def mhc_post_split_h( - x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor + x: torch.Tensor, + residual: torch.Tensor, + post: torch.Tensor, + comb: torch.Tensor, + *, + block_size: int | None = None, ) -> torch.Tensor: """Same result as the TileLang post kernel for contiguous BF16 HC=4 inputs.""" assert x.dtype == residual.dtype == torch.bfloat16 @@ -34,7 +39,7 @@ def mhc_post_split_h( assert residual.shape == (x.shape[0], 4, x.shape[1]) assert all(t.is_contiguous() for t in (x, residual, post, comb)) output = torch.empty_like(residual) - block = 128 if x.shape[0] <= 8 else 1024 + block = block_size if block_size is not None else (128 if x.shape[0] <= 8 else 1024) _mhc_post_split_h_kernel[(x.shape[0], triton.cdiv(x.shape[1], block))]( x, residual, diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py index 1422b8ff6f..15b6a831f2 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_draft_model.py @@ -482,8 +482,10 @@ def _block_quant_stack_applies(*, wkv_linears: list[torch.nn.Module]) -> bool: if not (block_quant and hasattr(quant_method, "w8a8_block_fp8_linear")): return False block_out = quant_method.quant_config.weight_block_size[0] + # the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout return all( linear.weight.dtype == torch.float8_e4m3fn + and linear.weight.dim() == 2 and linear.weight.shape[0] % block_out == 0 for linear in wkv_linears ) @@ -494,7 +496,8 @@ def _dequant_supported(linear: torch.nn.Module) -> bool: weight = linear.weight if weight.dtype in (torch.bfloat16, torch.float16, torch.float32): return True - if weight.dtype != torch.float8_e4m3fn: + # the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout + if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2: return False block = 128 out_dim, in_dim = weight.shape diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index c2cd5c0526..b109483f78 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -12,7 +12,7 @@ from sglang.srt.arg_groups.overrides import ( ) from sglang.srt.environ import envs from sglang.srt.runtime_context import get_platform -from sglang.srt.utils.common import is_npu +from sglang.srt.utils.common import is_gfx95_supported, is_npu if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -182,7 +182,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None: from sglang.srt.model_executor.cuda_graph_config import Backend incompatible = ( - ("non-CUDA hardware", not get_platform().is_cuda), + ( + "hardware other than CUDA or gfx950", + not ( + get_platform().is_cuda + or (get_platform().is_hip and is_gfx95_supported()) + ), + ), ( "prefill CUDA graphs", cfg.cuda_graph_config.prefill.backend != Backend.DISABLED, diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index c7a7553a10..aca25dcb88 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -92,7 +92,7 @@ class ExecFeatures(msgspec.Struct): enable_encoder_swa_bounded_replay: A[ bool, "DeepSeek-V4.1 encoder SWA bounded replay: cache Main KV and Indexer keys only, " - "rebuild request-owned SWA windows on prefix hits. Experimental; CUDA only.", + "rebuild request-owned SWA windows on prefix hits. Experimental; CUDA and gfx950.", ] = False enable_decoder_swa_bounded_replay: A[ bool, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d682610a3b..3b2434e614 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -922,7 +922,9 @@ class Envs: # output columns ride along nearly free. SGLANG_ROCM_K3_FUSE_KDA_INPROJ = EnvBool(True) SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256) - SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang") + # ROCm decode attention kernel: auto (aiter_sparse on gfx950, tilelang elsewhere) | + # aiter_sparse | tilelang | triton | torch | comparison | unified_kv_triton + SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("auto") SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False) SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index ce8d926134..a4a19b358d 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -2933,6 +2933,15 @@ class DeepseekV4AttnBackend( # out_loc is -1 for an incomplete group and 0 for padding; # the kernel suppresses both stores. assert out_loc is not None + if pool.low_ratio_index_k_is_split(layer_id): + # ROCm keeps the index-K payload and scales in two buffers + from sglang.srt.layers.attention.dsv4.low_ratio_backend_hip import ( + store_index_k_split, + ) + + return store_index_k_split( + pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id + ) index_k_norm_rope_pack_store( indexer.forward_wk(latent), indexer.k_norm.weight.data, diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index def42da7d1..91ec101b3a 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -4,13 +4,15 @@ import enum import functools import logging import os -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from typing import ( TYPE_CHECKING, + ClassVar, Dict, List, Literal, Optional, + Tuple, TypeVar, Union, ) @@ -18,23 +20,75 @@ from typing import ( import torch import torch.nn.functional as F +from sglang.kernels.ops.attention.dsv4.attn_glue_hip import ( + expand_index_page_table, + low_ratio_compression_metadata, + mask_indices_by_length, + page_table_from_req_to_token, + sparse_buffers, + widen_pair_i64, +) +from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( + CandidateBlocks, + slice_candidate_blocks, +) +from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( + HEAD_DIM as SWAPAB_HEAD_DIM, +) +from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( + MAX_BATCH as SWAPAB_MAX_BATCH, +) +from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( + NUM_HEADS as SWAPAB_NUM_HEADS, +) +from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( + SOFTMAX_SCALE as SWAPAB_SOFTMAX_SCALE, +) +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( init_compression_metadata as _init_compression_metadata_triton, ) +from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention +from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( + BuildCausalSwaPageIndices, + late_layer_tail_layout, +) from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + LateLayerTail, + _tail_rows, +) from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, FusedCompressMetadata, create_paged_compressor_data, ) +from sglang.srt.layers.attention.dsv4.dsv41_sparse import token_req_indices from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin +from sglang.srt.layers.attention.dsv4.low_ratio_backend_hip import ( + build_low_ratio_decode_workspaces, + low_ratio_candidate_span, + low_ratio_decode_rows_are_identity, + low_ratio_identity_skip_enabled, + low_ratio_index_topk_hip_decode, + low_ratio_index_topk_hip_extend, + refresh_low_ratio_prefill_workspaces, +) from sglang.srt.layers.attention.dsv4.metadata import ( PagedIndexerMetadata, copy_metadata, maybe_copy_inplace, ) +from sglang.srt.layers.attention.hip_flash_mla import ( + flash_mla_with_kvcache_entrypoint, + hip_attn_kv_splits, + resolve_hip_flashmla_backend, +) +from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( get_exec, @@ -74,6 +128,69 @@ def _pad_last_dim(x: T, multiples_of: int = PAGE_INDEX_ALIGNED_SIZE) -> T: return F.pad(x, pad=(0, target_size - curr_size), mode="constant", value=-1) +def _fold_lengths_into_index_lists( + indices: torch.Tensor, + lengths: Optional[torch.Tensor], + extra_indices: Optional[torch.Tensor] = None, + extra_lengths: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Entries of indices ([b, s, w]) at position >= lengths ([b]) set to -1 (the aiter + sparse kernel skips -1), likewise the optional second list; one launch for both.""" + if lengths is None: + indices, lengths = None, None + if extra_indices is not None and extra_lengths is None: + extra_indices = None + if indices is None and extra_indices is None: + return indices, extra_indices + first, first_len = ( + (indices, lengths) if indices is not None else (extra_indices, extra_lengths) + ) + second, second_len = ( + (extra_indices, extra_lengths) if indices is not None else (None, None) + ) + masked, masked_second = mask_indices_by_length( + first.contiguous(), first_len.contiguous(), second, second_len + ) + if indices is None: + return None, masked + return masked, masked_second if second is not None else extra_indices + + +def _fold_lengths_for_aiter_sparse( + core_attn_metadata, + compress_ratio: int, + swa_page_indices: torch.Tensor, + swa_topk_lengths: Optional[torch.Tensor], + extra_indices: Optional[torch.Tensor], + extra_topk_lengths: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """indices with positions >= lengths set to -1, since the aiter kernel takes + no per-row length; cached on the per-step metadata, once per step and ratio.""" + key = ( + compress_ratio, + swa_page_indices.data_ptr(), + tuple(swa_page_indices.shape), + None if extra_indices is None else extra_indices.data_ptr(), + None if extra_indices is None else tuple(extra_indices.shape), + ) + cache = core_attn_metadata._aiter_sparse_masked_indices + if cache is None: + cache = core_attn_metadata._aiter_sparse_masked_indices = {} + folded = cache.get(key) + if folded is None: + masked, masked_extra = _fold_lengths_into_index_lists( + swa_page_indices, swa_topk_lengths, extra_indices, extra_topk_lengths + ) + folded = ( + swa_page_indices if masked is None else masked, + masked_extra, + ) + # the C4 indexer rewrites c4_sparse_page_indices in place for every C4 layer + if compress_ratio != 4: + cache[key] = folded + return folded + + def _create_flashmla_metadata(): from sglang.srt.utils import is_hip @@ -211,10 +328,33 @@ class DSV4AttnMetadata: c128_topk_lengths_clamp1: Optional[torch.Tensor] = None c128_topk_lengths_raw: Optional[torch.Tensor] = None + # V4.1 ratios 1 / 2: latents live at raw_out_loc // ratio of the c1 / c2 pool; None when absent + low_ratios: Tuple[int, ...] = () + c1_out_loc: Optional[torch.Tensor] = None + c1_topk_lengths_clamp1: Optional[torch.Tensor] = None + c1_sparse_topk_lengths: Optional[torch.Tensor] = field(init=False, default=None) + c1_sparse_page_indices: Optional[torch.Tensor] = field(init=False, default=None) + c1_sparse_raw_indices: Optional[torch.Tensor] = field(init=False, default=None) + c2_out_loc: Optional[torch.Tensor] = None + c2_topk_lengths_clamp1: Optional[torch.Tensor] = None + c2_sparse_topk_lengths: Optional[torch.Tensor] = field(init=False, default=None) + c2_sparse_page_indices: Optional[torch.Tensor] = field(init=False, default=None) + c2_sparse_raw_indices: Optional[torch.Tensor] = field(init=False, default=None) + # unified-kv metadata unified: Optional[UnifiedKvMetadata] = None + request_window_layout: Optional[object] = None + + # length-folded aiter_sparse lists keyed by (ratio, address, shape); valid for one forward only + _aiter_sparse_masked_indices: Optional[dict] = field(default=None, repr=False) c0_flashmla_metadata: FlashMLASchedMeta = field(init=False, repr=False) + c1_flashmla_metadata: Optional[FlashMLASchedMeta] = field( + init=False, default=None, repr=False + ) + c2_flashmla_metadata: Optional[FlashMLASchedMeta] = field( + init=False, default=None, repr=False + ) c4_flashmla_metadata: FlashMLASchedMeta = field(init=False, repr=False) c128_flashmla_metadata: FlashMLASchedMeta = field(init=False, repr=False) @@ -222,9 +362,13 @@ class DSV4AttnMetadata: def positions(self) -> torch.Tensor: return self.positions_casual - def get_flashmla_metadata(self, compress_ratio: Literal[0, 4, 128]): + def get_flashmla_metadata(self, compress_ratio: Literal[0, 1, 2, 4, 128]): if compress_ratio == 0: return self.c0_flashmla_metadata + elif compress_ratio == 1: + return self.c1_flashmla_metadata + elif compress_ratio == 2: + return self.c2_flashmla_metadata elif compress_ratio == 4: return self.c4_flashmla_metadata elif compress_ratio == 128: @@ -232,6 +376,44 @@ class DSV4AttnMetadata: else: raise ValueError(f"invalid {compress_ratio=}") + def sparse_page_indices(self, compress_ratio: Literal[1, 2, 4]) -> torch.Tensor: + """Top-k slots into the ratio's extra cache, -1 padded; the indexer fills them.""" + if compress_ratio == 1: + return self.c1_sparse_page_indices + elif compress_ratio == 2: + return self.c2_sparse_page_indices + elif compress_ratio == 4: + return self.c4_sparse_page_indices + raise ValueError(f"invalid {compress_ratio=}") + + def drop_folded_sparse_indices(self, compress_ratio: Literal[1, 2, 4]) -> None: + """An index source rewrites sparse_page_indices(compress_ratio) in place, so the + length-folded copies keyed on that buffer stop describing it.""" + cache = self._aiter_sparse_masked_indices + if cache: + for key in [k for k in cache if k[0] == compress_ratio]: + del cache[key] + + def sparse_raw_indices( + self, compress_ratio: Literal[1, 2, 4] + ) -> Optional[torch.Tensor]: + if compress_ratio == 1: + return self.c1_sparse_raw_indices + elif compress_ratio == 2: + return self.c2_sparse_raw_indices + elif compress_ratio == 4: + return self.c4_sparse_raw_indices + raise ValueError(f"invalid {compress_ratio=}") + + def sparse_topk_lengths(self, compress_ratio: Literal[1, 2, 4]) -> torch.Tensor: + if compress_ratio == 1: + return self.c1_sparse_topk_lengths + elif compress_ratio == 2: + return self.c2_sparse_topk_lengths + elif compress_ratio == 4: + return self.c4_sparse_topk_lengths + raise ValueError(f"invalid {compress_ratio=}") + def copy_(self, other: DSV4AttnMetadata) -> None: copy_metadata( src=other, @@ -240,6 +422,7 @@ class DSV4AttnMetadata: "index_topk", "page_size", "cuda_int32_kwargs", + "low_ratios", ], copy_fields=[ "raw_out_loc", @@ -259,62 +442,69 @@ class DSV4AttnMetadata: "c4_sparse_topk_lengths_raw", "c4_sparse_page_indices", "c4_sparse_raw_indices", + "c1_out_loc", + "c1_topk_lengths_clamp1", + "c1_sparse_topk_lengths", + "c1_sparse_page_indices", + "c1_sparse_raw_indices", + "c2_out_loc", + "c2_topk_lengths_clamp1", + "c2_sparse_topk_lengths", + "c2_sparse_page_indices", + "c2_sparse_raw_indices", "unified", + "request_window_layout", ], assign_fields=[ # Recomputed by the recorded init_forward_metadata_in_graph op # each forward; not copied across replays. "swa_out_cache_loc", + "_aiter_sparse_masked_indices", "c0_flashmla_metadata", + "c1_flashmla_metadata", + "c2_flashmla_metadata", "c4_flashmla_metadata", "c128_flashmla_metadata", ], ) + # the captured store kernels read this by address, so a replay must find the live values at it + BREAKABLE_CUDA_GRAPH_IN_GRAPH_FIELDS: ClassVar[Tuple[str, ...]] = ( + "swa_out_cache_loc", + ) + def refresh_for_breakable_cuda_graph_replay_(self, other: DSV4AttnMetadata) -> None: - assert self.index_topk == other.index_topk + if not self.low_ratios: + assert self.page_size == other.page_size + assert self.index_topk == other.index_topk + assert self.low_ratios == other.low_ratios + for f in fields(self): + src, dst = getattr(other, f.name), getattr(self, f.name) + if torch.is_tensor(src) or torch.is_tensor(dst): + assert src is not None and dst is not None, f.name + dst.copy_(src) + elif f.name == "unified" and src is not None: + assert dst is not None + dst.refresh_for_breakable_cuda_graph_replay_(src) + else: + setattr(self, f.name, src) + return + # low ratios: rebind every field to other's (built for the live batch), except the + # SWA store target, which the captured segments read by address and is copied assert self.page_size == other.page_size - assert self.cuda_int32_kwargs == other.cuda_int32_kwargs - - tensor_copy_fields = [ - "raw_out_loc", - "seq_lens_casual", - "positions_casual", - "swa_out_cache_loc", - "c4_out_loc", - "c128_out_loc", - "page_table", - "swa_page_indices", - "swa_topk_lengths", - "c128_page_indices", - "c128_topk_lengths_clamp1", - "c128_topk_lengths_raw", - "c4_topk_lengths_raw", - "c4_topk_lengths_clamp1", - "c4_sparse_topk_lengths", - "c4_sparse_topk_lengths_raw", - "c4_sparse_page_indices", - "c4_sparse_raw_indices", - ] - for field_name in tensor_copy_fields: - src_val = getattr(other, field_name) - dst_val = getattr(self, field_name) - if src_val is None and dst_val is None: - continue - assert src_val is not None and dst_val is not None, ( - f"{field_name=} {src_val=} {dst_val=}" - ) - dst_val.copy_(src_val) - - if self.unified is None and other.unified is None: - pass - else: - assert self.unified is not None and other.unified is not None - self.unified.refresh_for_breakable_cuda_graph_replay_(other.unified) - - self.c0_flashmla_metadata = other.c0_flashmla_metadata - self.c4_flashmla_metadata = other.c4_flashmla_metadata - self.c128_flashmla_metadata = other.c128_flashmla_metadata + assert self.index_topk == other.index_topk + assert self.low_ratios == other.low_ratios + for f in fields(self): + name = f.name + src = getattr(other, name) + if name in self.BREAKABLE_CUDA_GRAPH_IN_GRAPH_FIELDS: + dst = getattr(self, name) + assert dst is not None and src is not None, f"{name=} {dst=} {src=}" + assert dst.shape == src.shape, f"{name=} {dst.shape=} {src.shape=}" + dst.copy_(src) + else: + # the length-fold cache too: a stale entry could match a reused address + setattr(self, name, src) def init_compression_metadata(self, unified_swa_pages: int = 0): assert self.page_table.dim() == 2 @@ -350,6 +540,12 @@ class DSV4AttnMetadata: self.unified.c4_out_loc = self.c4_out_loc + unified_swa_pages self.unified.c128_out_loc = self.c128_out_loc + unified_swa_pages + # ratio 1/2 counterparts of the c4/c128 metadata, one launch for both + for name, value in low_ratio_compression_metadata( + self.seq_lens_casual, self.raw_out_loc, self.low_ratios + ).items(): + setattr(self, name, value) + _CP_REINDEX_FIELDS = [ "seq_lens_casual", "positions_casual", @@ -407,25 +603,32 @@ class DSV4AttnMetadata: "supported: 512 (small) or 1024 (large)" ) assert self.c4_topk_lengths_clamp1 is not None - self.c4_sparse_topk_lengths = torch.clamp( - self.c4_topk_lengths_clamp1, max=self.index_topk - ) assert self.c4_topk_lengths_raw is not None - self.c4_sparse_topk_lengths_raw = torch.clamp( - self.c4_topk_lengths_raw, max=self.index_topk - ) - self.c4_sparse_page_indices = torch.full( - (self.c4_topk_lengths_clamp1.size(0), self.index_topk), - -1, - dtype=torch.int32, - device=self.c4_topk_lengths_clamp1.device, - ) - self.c4_sparse_page_indices = _pad_last_dim(self.c4_sparse_page_indices) + # the clamped lengths and the -1 filled top-k buffers of every ratio in one launch + for name, value in sparse_buffers( + c4_topk_lengths_clamp1=self.c4_topk_lengths_clamp1, + c4_topk_lengths_raw=self.c4_topk_lengths_raw, + c1_topk_lengths_clamp1=( + self.c1_topk_lengths_clamp1 if 1 in self.low_ratios else None + ), + c2_topk_lengths_clamp1=( + self.c2_topk_lengths_clamp1 if 2 in self.low_ratios else None + ), + index_topk=self.index_topk, + page_index_align=PAGE_INDEX_ALIGNED_SIZE, + ).items(): + setattr(self, name, value) if is_prefill: self.c4_sparse_raw_indices = torch.empty_like(self.c4_sparse_page_indices) self.c0_flashmla_metadata = _create_flashmla_metadata() self.c4_flashmla_metadata = _create_flashmla_metadata() self.c128_flashmla_metadata = _create_flashmla_metadata() + if 1 in self.low_ratios: + self.c1_sparse_raw_indices = torch.empty_like(self.c1_sparse_page_indices) + self.c1_flashmla_metadata = _create_flashmla_metadata() + if 2 in self.low_ratios: + self.c2_sparse_raw_indices = torch.empty_like(self.c2_sparse_page_indices) + self.c2_flashmla_metadata = _create_flashmla_metadata() @dataclass @@ -433,8 +636,24 @@ class DSV4Metadata: core_attn_metadata: DSV4AttnMetadata indexer_metadata: Optional[PagedIndexerMetadata] + # low-ratio indexer metadata per ratio: decode rows per request (clamp-1), prefill per token + c1_indexer_metadata: Optional[PagedIndexerMetadata] = None + c2_indexer_metadata: Optional[PagedIndexerMetadata] = None + c4_compress_metadata: Optional[FusedCompressMetadata] = None c128_compress_metadata: Optional[FusedCompressMetadata] = None + # per-step inputs of forward_low_ratio_sources, hoisted for one-token rows and the replay tail + low_ratio_req_indices: Optional[torch.Tensor] = None + low_ratio_pos_i64: Optional[torch.Tensor] = None + # Per-step scratch for the TP-padded query heads (models/deepseek_v4.py). + q_pad_buffer: Optional[torch.Tensor] = None + # low-ratio FlyDSL indexer workspaces by ratio; address-pinned like the c4 ones, rebuilt in place + fp4_low_ratio_decode_workspaces: Dict[int, FP4DecodeWorkspace] = field( + default_factory=dict, repr=False + ) + fp4_low_ratio_prefill_workspaces: Dict[int, FP4PrefillWorkspace] = field( + default_factory=dict, repr=False + ) # Captured kernels bind these FP4 indexer buffers by address; absent from copy_ # so addresses stay pinned across replays (the builders refresh contents). fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False) @@ -446,49 +665,85 @@ class DSV4Metadata: # AITER's rope kernels require int64 positions while the core metadata keeps # them int32, so widen once per forward instead of once per C4 layer. fp4_q_positions: Optional[torch.Tensor] = field(default=None, repr=False) + # only on the metadata built for the late layers under decoder SWA bounded replay + late_layer_tail: Optional[LateLayerTail] = None @property def core_metadata(self) -> DSV4AttnMetadata: return self.core_attn_metadata + def low_ratio_indexer_metadata( + self, compress_ratio: int + ) -> Optional[PagedIndexerMetadata]: + return ( + self.c1_indexer_metadata + if compress_ratio == 1 + else self.c2_indexer_metadata + ) + + def low_ratio_indexer_metadata_by_ratio(self) -> Dict[int, PagedIndexerMetadata]: + out = {} + if self.c1_indexer_metadata is not None: + out[1] = self.c1_indexer_metadata + if self.c2_indexer_metadata is not None: + out[2] = self.c2_indexer_metadata + return out + def copy_(self, other: DSV4Metadata): self.core_attn_metadata.copy_(other.core_attn_metadata) maybe_copy_inplace(self.indexer_metadata, src=other.indexer_metadata) + maybe_copy_inplace(self.c1_indexer_metadata, src=other.c1_indexer_metadata) + maybe_copy_inplace(self.c2_indexer_metadata, src=other.c2_indexer_metadata) maybe_copy_inplace(self.c4_compress_metadata, src=other.c4_compress_metadata) maybe_copy_inplace( self.c128_compress_metadata, src=other.c128_compress_metadata ) + # allocated inside a captured segment (graph-pool storage), so a refresh must keep it + BREAKABLE_CUDA_GRAPH_KEEP_FIELDS: ClassVar[Tuple[str, ...]] = ("q_pad_buffer",) + def refresh_for_breakable_cuda_graph_replay_(self, other: DSV4Metadata) -> None: self.core_attn_metadata.refresh_for_breakable_cuda_graph_replay_( other.core_attn_metadata ) - maybe_copy_inplace(self.indexer_metadata, src=other.indexer_metadata) - maybe_copy_inplace(self.c4_compress_metadata, src=other.c4_compress_metadata) - maybe_copy_inplace( - self.c128_compress_metadata, src=other.c128_compress_metadata - ) - - if self.fp4_k_write_metadata is None and other.fp4_k_write_metadata is None: - pass - else: - assert ( - self.fp4_k_write_metadata is not None - and other.fp4_k_write_metadata is not None + if not self.core_attn_metadata.low_ratios: + maybe_copy_inplace(self.indexer_metadata, src=other.indexer_metadata) + maybe_copy_inplace( + self.c4_compress_metadata, src=other.c4_compress_metadata + ) + maybe_copy_inplace( + self.c128_compress_metadata, src=other.c128_compress_metadata ) - for captured, replay in zip( - self.fp4_k_write_metadata, - other.fp4_k_write_metadata, - strict=True, - ): - captured.copy_(replay) - if self.fp4_q_positions is None and other.fp4_q_positions is None: - pass - else: - assert self.fp4_q_positions is not None - assert other.fp4_q_positions is not None - self.fp4_q_positions.copy_(other.fp4_q_positions) + if self.fp4_k_write_metadata is None and other.fp4_k_write_metadata is None: + pass + else: + assert ( + self.fp4_k_write_metadata is not None + and other.fp4_k_write_metadata is not None + ) + for captured, replay in zip( + self.fp4_k_write_metadata, + other.fp4_k_write_metadata, + strict=True, + ): + captured.copy_(replay) + + if self.fp4_q_positions is None and other.fp4_q_positions is None: + pass + else: + assert self.fp4_q_positions is not None + assert other.fp4_q_positions is not None + self.fp4_q_positions.copy_(other.fp4_q_positions) + return + for f in fields(self): + name = f.name + if ( + name == "core_attn_metadata" + or name in self.BREAKABLE_CUDA_GRAPH_KEEP_FIELDS + ): + continue + setattr(self, name, getattr(other, name)) @dataclass @@ -554,13 +809,32 @@ class _GraphBucket(enum.Enum): class DeepseekV4HipRadixBackend( - AttentionBackend, C4IndexerBackendMixin, CompressorBackendMixin + AttentionBackend, + C4IndexerBackendMixin, + CompressorBackendMixin, ): # DSV4 TBO runs only in eager prefill; driving TBO children through graph # capture/replay leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES). tbo_supports_cuda_graph = False + supports_prefill_cuda_graph_max_context_size = True supports_ragged_verify_graph: bool = True + # each bucket keeps one metadata object; the captured segments read the SWA store target by address use_captured_forward_metadata_for_breakable_cuda_graph: bool = True + + forward = DeepseekV4AttnBackend.forward + + # the ratio-1/2 compressor and indexer orchestration is the CUDA backend's, taken + # unbound; HIP differs only in the paged top-k (_low_ratio_index_topk below) + forward_low_ratio_sources = DeepseekV4AttnBackend.forward_low_ratio_sources + _forward_low_ratio_sources_cp = DeepseekV4AttnBackend._forward_low_ratio_sources_cp + low_ratio_prefill_graph = DeepseekV4AttnBackend.low_ratio_prefill_graph + _low_ratio_in_prefill_graph = DeepseekV4AttnBackend._low_ratio_in_prefill_graph + _low_ratio_compress = DeepseekV4AttnBackend._low_ratio_compress + _low_ratio_compress_decode = DeepseekV4AttnBackend._low_ratio_compress_decode + _low_ratio_compress_torch = DeepseekV4AttnBackend._low_ratio_compress_torch + _low_ratio_pair_partners = DeepseekV4AttnBackend._low_ratio_pair_partners + _low_ratio_write_group = DeepseekV4AttnBackend._low_ratio_write_group + _low_ratio_compress_fused = DeepseekV4AttnBackend._low_ratio_compress_fused # MIXED BCG replay regresses ROCm DSV4 DP-attention serving throughput. prefer_eager_mixed_prefill_under_dp_attention: bool = True @@ -574,6 +848,7 @@ class DeepseekV4HipRadixBackend( ): super().__init__() self.device = torch.device(model_runner.device) + self.max_context_len = model_runner.model_config.context_len head_dim = model_runner.model_config.head_dim assert head_dim == 512, ( "DSV4 MQA head_dim = qk_nope_head_dim(448) + qk_rope_head_dim(64) = 512" @@ -595,19 +870,67 @@ class DeepseekV4HipRadixBackend( self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1] assert isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool) + # index_topk and candidate_* are optional HF-config keys self.index_topk = getattr( model_runner.model_config.hf_text_config, "index_topk", DEFAULT_INDEX_TOPK ) + # the distinct ratios this pool has, sorted: (4, 128) for V4, (1, 2) for V4.1 + self.present_ratios = self.token_to_kv_pool.present_ratios + self.low_ratios: Tuple[int, ...] = tuple( + ratio for ratio in (1, 2) if ratio in self.present_ratios + ) + self.has_c4: bool = 4 in self.present_ratios + self.has_c128: bool = 128 in self.present_ratios + # rows whose compressed context fits index_topk take the sequential selection, unscored + hf_text_config = model_runner.model_config.hf_text_config + self.low_ratio_identity_skip: bool = bool( + self.low_ratios + ) and low_ratio_identity_skip_enabled( + index_topk=self.index_topk, + candidate_topk_blocks=getattr(hf_text_config, "candidate_topk_blocks", 0), + candidate_block_size=getattr(hf_text_config, "candidate_block_size", 1), + ) + # a decode batch whose longest context fits this span skips the candidate-block filter + self.low_ratio_candidate_span: Optional[int] = ( + low_ratio_candidate_span(hf_text_config) if self.low_ratios else None + ) + # published by the candidate-source layer, consumed by the later index-source layers + self.candidate_masks = None self.enable_deepseek_v4_fp4_indexer: bool = ( get_exec().kernel.enable_deepseek_v4_fp4_indexer ) + self.enable_decoder_swa_bounded_replay: bool = ( + get_exec().features.enable_decoder_swa_bounded_replay + ) + if self.enable_decoder_swa_bounded_replay: + # both read the store target off the full batch, not the tail metadata + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_triton, + ) + + if is_unified_kv_triton(): + raise NotImplementedError( + "decoder SWA bounded replay is not wired for unified_kv_triton" + ) + # the model switches onto it after the last kv_source layer (enter_late_layer_tail) + self.tail_forward_metadata: Optional[DSV4Metadata] = None + self.encoder_replay = False self.topk = get_spec().speculative_eagle_topk or 0 assert self.topk in [0, 1], "MTP Topk > 1 not supported for DeepSeek V4" self.mtp_enabled = self.topk > 0 self.speculative_num_steps = speculative_num_steps self.speculative_num_draft_tokens: int = get_spec().speculative_num_draft_tokens self.is_draft_worker = getattr(model_runner, "is_draft_worker", False) + self._verify_mask: Optional[VerifyMask] = None self.is_dspark = model_runner.spec_algorithm.is_dspark() + # the DSpark draft reads the target's checkpoint, so its config names V4.1 too + self.is_dsv41 = ( + getattr(model_runner.model_config.hf_text_config, "model_type", None) + == "deepseek_v41" + ) + # Decode and speculative metadata can be rebuilt from device lengths. + # The online c128 planner still consumes CPU lengths. + self.needs_cpu_seq_lens = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() self.is_dspark_draft = self.is_draft_worker and self.is_dspark # Draft layers are all COMPRESS_RATIO_NEXTN_LAYER (0), so the draft pool # has neither compressed kv nor an indexer pool. @@ -639,15 +962,64 @@ class DeepseekV4HipRadixBackend( pin_tensor = torch.tensor(x, dtype=torch.int32, pin_memory=True) return pin_tensor.to(self.device, non_blocking=True) - def init_forward_metadata_indexer(self, core_attn_metadata: DSV4AttnMetadata): + def init_forward_metadata_indexer( + self, + core_attn_metadata: DSV4AttnMetadata, + *, + compress_ratio: int = 4, + is_prefill: bool = False, + ): + page_table = core_attn_metadata.page_table + index_page_size = 0 + if compress_ratio == 4: + c_seq_lens = core_attn_metadata.c4_topk_lengths_raw + elif compress_ratio in (1, 2): + if is_prefill: + # raw visible count per token row; a zero row is skipped by the prefill kernel + c_seq_lens = (core_attn_metadata.seq_lens_casual // compress_ratio).to( + torch.int32 + ) + else: + c_seq_lens = ( + core_attn_metadata.c1_topk_lengths_clamp1 + if compress_ratio == 1 + else core_attn_metadata.c2_topk_lengths_clamp1 + ) + # the low-ratio indexer-K pool pages at 64 slots, not page_size // ratio + index_page_size = self.token_to_kv_pool.get_index_k_page_size( + compress_ratio + ) + slots_per_page = self.page_size // compress_ratio + assert slots_per_page % index_page_size == 0, ( + f"{self.page_size = } / {compress_ratio = } must be a multiple of " + f"{index_page_size = }" + ) + page_table = expand_index_page_table( + page_table, slots_per_page // index_page_size + ) + else: + raise ValueError(f"Unsupported indexer {compress_ratio = }") return PagedIndexerMetadata( page_size=self.page_size, - compressed_page_size=self.token_to_kv_pool.get_index_k_page_size(), - page_table=core_attn_metadata.page_table, - compressed_seq_lens=core_attn_metadata.c4_topk_lengths_raw, + compressed_page_size=self.token_to_kv_pool.get_index_k_page_size( + compress_ratio + ), + page_table=page_table, + compressed_seq_lens=c_seq_lens, use_topk_v2=self.dsa_topk_backend.should_use_topk_v2(), + compress_ratio=compress_ratio, ) + def _init_low_ratio_indexer_metadata( + self, core_attn_metadata: DSV4AttnMetadata, *, is_prefill: bool + ) -> Dict[str, PagedIndexerMetadata]: + return { + f"c{ratio}_indexer_metadata": self.init_forward_metadata_indexer( + core_attn_metadata, compress_ratio=ratio, is_prefill=is_prefill + ) + for ratio in core_attn_metadata.low_ratios + } + def init_forward_metadata_decode( self, max_seq_len: int, @@ -670,16 +1042,18 @@ class DeepseekV4HipRadixBackend( max_seq_len: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, - seq_lens_cpu: List[int], + seq_lens_cpu: Optional[List[int]], out_cache_loc: torch.Tensor, num_tokens: int, extend_seq_lens: torch.Tensor, - extend_seq_lens_cpu: List[int], + extend_seq_lens_cpu: Optional[List[int]], need_compress: bool = True, use_prefill_cuda_graph: bool = False, compress_gpu_plan: bool = False, extend_start_loc: Optional[torch.Tensor] = None, attach_decode_streams: bool = False, + low_ratio_decode_rows: bool = False, + swa_replay_start: Optional[torch.Tensor] = None, # Whether num_tokens == sum(extend_seq_lens) exactly, which lets the # token map skip an implicit D2H. exact_num_tokens: bool = True, @@ -710,6 +1084,8 @@ class DeepseekV4HipRadixBackend( out_loc=out_cache_loc, need_compress=need_compress, is_prefill=True, + swa_replay_start=swa_replay_start, + num_groups=len(req_pool_indices), ) # Normal prefill starts with a conservative exact_num_tokens=False. # Its CPU length mirror proves the exact query count without a D2H sync. @@ -738,7 +1114,7 @@ class DeepseekV4HipRadixBackend( ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) - if need_compress + if need_compress and self.has_c4 else None ) if not need_compress: @@ -778,11 +1154,27 @@ class DeepseekV4HipRadixBackend( use_prefill_cuda_graph=False, ) + low_ratio_indexer_metadata = ( + self._init_low_ratio_indexer_metadata( + core_attn_metadata, is_prefill=not low_ratio_decode_rows + ) + if need_compress + else {} + ) return DSV4Metadata( core_attn_metadata, indexer_metadata, - c4_compress_metadata=create(compress_ratio=4), - c128_compress_metadata=create(compress_ratio=128), + c4_compress_metadata=( + create(compress_ratio=4) + if self.has_c4 + else _create_dummy_paged_compress_data(compress_ratio=4) + ), + c128_compress_metadata=( + create(compress_ratio=128) + if self.has_c128 + else _create_dummy_paged_compress_data(compress_ratio=128) + ), + **low_ratio_indexer_metadata, ) def init_forward_metadata_target_verify( @@ -816,8 +1208,6 @@ class DeepseekV4HipRadixBackend( seq_lens=seq_lens, out_cache_loc=out_cache_loc, ) - if seq_lens_cpu is None: - seq_lens_cpu = seq_lens.tolist() return self.init_forward_metadata_target_verify_old( max_seq_len=max_seq_len, req_pool_indices=req_pool_indices, @@ -865,12 +1255,18 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu = None else: seq_lens = seq_lens + self.target_verify_num_draft_tokens - seq_lens_cpu = [ - x + self.target_verify_num_draft_tokens for x in seq_lens_cpu - ] - extend_seq_lens_cpu = [self.target_verify_num_draft_tokens] * batch_size + if seq_lens_cpu is None: + extend_seq_lens_cpu = None + extend_seq_lens = torch.full_like( + seq_lens, self.target_verify_num_draft_tokens + ) + else: + seq_lens_cpu = [ + x + self.target_verify_num_draft_tokens for x in seq_lens_cpu + ] + extend_seq_lens_cpu = [self.target_verify_num_draft_tokens] * batch_size + extend_seq_lens = self._move_to_device(extend_seq_lens_cpu) num_tokens = self.target_verify_num_draft_tokens * batch_size - extend_seq_lens = self._move_to_device(extend_seq_lens_cpu) exact_num_tokens = True if out_cache_loc is None: out_cache_loc = seq_lens.new_zeros(num_tokens) @@ -889,10 +1285,70 @@ class DeepseekV4HipRadixBackend( # guarded inside init_forward_metadata_prefill, not here need_compress=self.need_compress, use_prefill_cuda_graph=use_prefill_cuda_graph, - compress_gpu_plan=ragged_layout is not None, + compress_gpu_plan=seq_lens_cpu is None, extend_start_loc=extend_start_loc, attach_decode_streams=True, exact_num_tokens=exact_num_tokens, + low_ratio_decode_rows=True, + ) + + def _uses_dspark_draft_window(self) -> bool: + """Whether the V4.1 DSpark draft block takes the block-window metadata; DSv4's + draft keeps the causal verify indices, and unified_kv addresses its ring by + (slot, position) and keeps its own path.""" + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_triton, + ) + + return self.is_dspark_draft and self.is_dsv41 and not is_unified_kv_triton() + + def init_forward_metadata_dspark_draft_block( + self, + max_seq_len: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + out_cache_loc: torch.Tensor, + block_size: int, + ) -> DSV4Metadata: + """DSpark draft block: seq_lens are the committed prefixes, extended by + block_size per request from the device lengths only (the worker's + seq_lens_cpu is already extended). SWA-only.""" + assert block_size == self.target_verify_num_draft_tokens, ( + f"{block_size=} must equal the draft's target_verify_num_draft_tokens=" + f"{self.target_verify_num_draft_tokens}" + ) + batch_size = len(seq_lens) + num_tokens = block_size * batch_size + assert out_cache_loc.shape[0] >= num_tokens, ( + f"{out_cache_loc.shape=} holds fewer than {num_tokens=} block slots" + ) + seq_lens_casual, req_pool_indices_repeated = ( + self.expand_extend_with_same_length( + bs=batch_size, + qo_len=block_size, + seq_lens=seq_lens + block_size, + req_pool_indices=req_pool_indices, + ) + ) + core_attn_metadata = self.make_core_attn_metadata( + req_to_token=self.req_to_token, + req_pool_indices_repeated=req_pool_indices_repeated, + seq_lens_casual=seq_lens_casual, + # The block may cross into the next page beyond the prefix. + max_seq_len=max_seq_len + block_size, + out_loc=out_cache_loc, + need_compress=False, + is_prefill=True, + dspark_block_size=block_size, + num_groups=batch_size, + ) + return DSV4Metadata( + core_attn_metadata, + None, + c4_compress_metadata=_create_dummy_paged_compress_data(compress_ratio=4), + c128_compress_metadata=_create_dummy_paged_compress_data( + compress_ratio=128 + ), ) def make_forward_metadata_from_raw_verify( @@ -926,6 +1382,7 @@ class DeepseekV4HipRadixBackend( max_seq_len=self.MAX_SEQ_LEN_FOR_CAPTURE, out_loc=out_cache_loc, need_compress=need_compress, + num_groups=bs, ) # extend_seq_lens is uniform here (seq_lens already carries the draft # block, so the minimum above cannot trim it), hence an exact token count. @@ -944,7 +1401,7 @@ class DeepseekV4HipRadixBackend( ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) - if need_compress + if need_compress and self.has_c4 else None ) if not need_compress: @@ -966,8 +1423,19 @@ class DeepseekV4HipRadixBackend( return DSV4Metadata( core_attn_metadata, indexer_metadata, - c4_compress_metadata=create(compress_ratio=4), - c128_compress_metadata=create(compress_ratio=128), + **self._init_low_ratio_indexer_metadata( + core_attn_metadata, is_prefill=False + ), + c4_compress_metadata=( + create(compress_ratio=4) + if self.has_c4 + else _create_dummy_paged_compress_data(compress_ratio=4) + ), + c128_compress_metadata=( + create(compress_ratio=128) + if self.has_c128 + else _create_dummy_paged_compress_data(compress_ratio=128) + ), ) def make_forward_metadata_from_raw_decode( @@ -993,9 +1461,12 @@ class DeepseekV4HipRadixBackend( self._attach_unified_kv_decode_streams(core_attn_metadata, req_pool_indices) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) - if need_compress + if need_compress and self.has_c4 else None ) + low_ratio_indexer_metadata = self._init_low_ratio_indexer_metadata( + core_attn_metadata, is_prefill=False + ) if not need_compress: create = _create_dummy_paged_compress_data @@ -1012,8 +1483,17 @@ class DeepseekV4HipRadixBackend( return DSV4Metadata( core_attn_metadata, indexer_metadata, - c4_compress_metadata=create(compress_ratio=4), - c128_compress_metadata=create(compress_ratio=128), + c4_compress_metadata=( + create(compress_ratio=4) + if self.has_c4 + else _create_dummy_paged_compress_data(compress_ratio=4) + ), + c128_compress_metadata=( + create(compress_ratio=128) + if self.has_c128 + else _create_dummy_paged_compress_data(compress_ratio=128) + ), + **low_ratio_indexer_metadata, ) def init_forward_metadata_draft_extend( @@ -1021,20 +1501,24 @@ class DeepseekV4HipRadixBackend( max_seq_len: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, - seq_lens_cpu: List[int], + seq_lens_cpu: Optional[List[int]], num_tokens_per_req: int, out_cache_loc: Optional[torch.Tensor] = None, use_prefill_cuda_graph: bool = False, ) -> DSV4Metadata: batch_size = len(seq_lens) - extend_seq_lens_cpu = [num_tokens_per_req] * batch_size - extend_seq_lens = self._move_to_device(extend_seq_lens_cpu) + extend_seq_lens_cpu = ( + [num_tokens_per_req] * batch_size if seq_lens_cpu is not None else None + ) + extend_seq_lens = torch.full_like(seq_lens, num_tokens_per_req) num_tokens = num_tokens_per_req * batch_size if out_cache_loc is None: out_cache_loc = seq_lens.new_zeros(num_tokens) return self.init_forward_metadata_prefill( seq_lens=seq_lens, - max_seq_len=max_seq_len, + # Draft extend attends only to SWA; compressed attention never reads + # this page-table placeholder. Match CUDA's one-page allocation. + max_seq_len=self.page_size, req_pool_indices=req_pool_indices, seq_lens_cpu=seq_lens_cpu, out_cache_loc=out_cache_loc, @@ -1057,13 +1541,22 @@ class DeepseekV4HipRadixBackend( raw_metadata=self.forward_metadata, ) + metadata = self.forward_metadata + if isinstance(metadata, DSV4Metadata): + # the bucket object outlives the step; a warmup's length-fold cache must not be replayed + metadata.core_attn_metadata._aiter_sparse_masked_indices = None + + window = self.token_to_kv_pool.request_window + if window is not None and isinstance(metadata, DSV4Metadata): + window.activate(metadata.core_attn_metadata.request_window_layout) + # Spec-v2 and DP padding can rebind out_cache_loc after out-graph prep; # capture the translation here so replay reads live locations. # FlashMLA requires int32 indices. - metadata = self.forward_metadata if ( isinstance(metadata, DSV4Metadata) and forward_batch.out_cache_loc is not None + and window is None ): out_cache_loc = forward_batch.out_cache_loc if ( @@ -1103,6 +1596,46 @@ class DeepseekV4HipRadixBackend( # The out-graph refresh saw raw metadata and skipped; otherwise the # logits kernel builds a schedule that frees scratch replays re-read. self._refresh_fp4_prefill_workspace(forward_batch) + # target-verify rows are one token each, so they take the decode indexer's inputs + one_token_rows = ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ) + if ( + isinstance(metadata, DSV4Metadata) + and metadata.core_metadata.low_ratios + and one_token_rows + ): + if ( + forward_batch.forward_mode.is_decode() + and forward_batch.positions.is_cuda + and forward_batch.req_pool_indices.shape + == forward_batch.positions.shape + ): + # both widenings in one launch + metadata.low_ratio_req_indices, metadata.low_ratio_pos_i64 = ( + widen_pair_i64( + forward_batch.req_pool_indices, forward_batch.positions + ) + ) + else: + metadata.low_ratio_req_indices = token_req_indices(forward_batch) + metadata.low_ratio_pos_i64 = forward_batch.positions.to(torch.int64) + # same capture-safe workspace contract as the c4 ones below + metadata.fp4_low_ratio_decode_workspaces = ( + build_low_ratio_decode_workspaces( + { + ratio: indexer_metadata + for ratio, indexer_metadata in ( + metadata.low_ratio_indexer_metadata_by_ratio().items() + ) + # Identity rows are not scored, so they need no schedule. + if not low_ratio_decode_rows_are_identity( + self, forward_batch, ratio + ) + } + ) + ) # Decode's schedule builder is capture-safe (the workspace pins its scratch); # prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. @@ -1130,10 +1663,25 @@ class DeepseekV4HipRadixBackend( and metadata.c4_compress_metadata is not None ) - def _refresh_fp4_prefill_workspace(self, forward_batch: ForwardBatch) -> None: + def _refresh_fp4_prefill_workspace( + self, forward_batch: ForwardBatch, metadata: Optional[DSV4Metadata] = None + ) -> None: """Must run outside graph capture: AITER's scheduler frees the scratch its kernel reads, so only the pinned buffers it fills may be read from a graph.""" - metadata = self.forward_metadata + if metadata is None: + metadata = self.forward_metadata + if ( + isinstance(metadata, DSV4Metadata) + and metadata.core_metadata.low_ratios + and forward_batch.forward_mode in (ForwardMode.EXTEND, ForwardMode.MIXED) + and not torch.cuda.is_current_stream_capturing() + ): + metadata.fp4_low_ratio_prefill_workspaces = ( + refresh_low_ratio_prefill_workspaces( + metadata.low_ratio_indexer_metadata_by_ratio(), + metadata.fp4_low_ratio_prefill_workspaces, + ) + ) if not self._fp4_workspaces_enabled(metadata): return if forward_batch.forward_mode not in ( @@ -1181,14 +1729,12 @@ class DeepseekV4HipRadixBackend( else: out_cache_loc = None actual_forward_mode = forward_batch.forward_mode - seq_lens_sum = int(seq_lens.sum().item()) - seq_lens_cpu = seq_lens.cpu() + seq_lens_cpu = seq_lens.cpu() if self.needs_cpu_seq_lens else None else: out_cache_loc = forward_batch.out_cache_loc actual_forward_mode = getattr( forward_batch, "actual_forward_mode", forward_batch.forward_mode ) - seq_lens_sum = forward_batch.seq_lens_sum seq_lens_cpu = forward_batch.seq_lens_cpu if actual_forward_mode == ForwardMode.IDLE: @@ -1200,20 +1746,17 @@ class DeepseekV4HipRadixBackend( device = seq_lens.device seq_lens = torch.ones(bs, dtype=seq_lens.dtype, device=device) seq_lens_cpu = torch.ones(bs, dtype=torch.int64) - seq_lens_sum = bs req_pool_indices = torch.zeros( bs, dtype=req_pool_indices.dtype, device=device ) out_cache_loc = torch.zeros(bs, dtype=torch.int64, device=device) - assert seq_lens_cpu is not None seq_lens = seq_lens[:bs] - seq_lens_cpu = seq_lens_cpu[:bs] req_pool_indices = req_pool_indices[:bs] - - actual_max_seq_len = seq_lens_cpu.max().item() chosen_max_seq_len = self.MAX_SEQ_LEN_FOR_CAPTURE - assert actual_max_seq_len <= chosen_max_seq_len + if seq_lens_cpu is not None: + seq_lens_cpu = seq_lens_cpu[:bs] + assert seq_lens_cpu.max().item() <= chosen_max_seq_len graph_key = bs @@ -1232,6 +1775,23 @@ class DeepseekV4HipRadixBackend( seq_lens=seq_lens, out_cache_loc=out_cache_loc_padded, ) + elif bucket == _GraphBucket.TARGET_VERIFY and self._uses_dspark_draft_window(): + assert out_cache_loc is not None + block_size = self.target_verify_num_draft_tokens + num_tokens_v = block_size * bs + out_cache_loc_padded = F.pad( + out_cache_loc, + pad=(0, num_tokens_v - len(out_cache_loc)), + mode="constant", + value=0, + ) + temp_metadata = self.init_forward_metadata_dspark_draft_block( + max_seq_len=chosen_max_seq_len, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + out_cache_loc=out_cache_loc_padded, + block_size=block_size, + ) elif bucket == _GraphBucket.TARGET_VERIFY: assert out_cache_loc is not None ragged_layout = resolve_ragged_verify_layout(forward_batch) @@ -1255,7 +1815,9 @@ class DeepseekV4HipRadixBackend( use_prefill_cuda_graph=True, # CPU mirror already available here (== seq_lens, no D2H); # pass it so target_verify skips the per-iter seq_lens.tolist() sync. - seq_lens_cpu=seq_lens_cpu.tolist(), + seq_lens_cpu=( + seq_lens_cpu.tolist() if seq_lens_cpu is not None else None + ), ragged_layout=ragged_layout, ) elif bucket == _GraphBucket.DRAFT_EXTEND: @@ -1273,7 +1835,9 @@ class DeepseekV4HipRadixBackend( max_seq_len=chosen_max_seq_len, req_pool_indices=req_pool_indices, seq_lens=seq_lens, - seq_lens_cpu=seq_lens_cpu.tolist(), + seq_lens_cpu=( + seq_lens_cpu.tolist() if seq_lens_cpu is not None else None + ), num_tokens_per_req=num_tokens_per_req, out_cache_loc=out_cache_loc, use_prefill_cuda_graph=True, @@ -1304,18 +1868,25 @@ class DeepseekV4HipRadixBackend( max_seq_len_override: Optional[int] = None, use_prefill_cuda_graph: bool = False, ): + self.encoder_replay = forward_batch.encoder_swa_replay req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens.to(torch.int32) seq_lens_cpu = forward_batch.seq_lens_cpu assert self.req_to_token_pool.req_to_token is self.req_to_token assert self.page_size % 128 == 0 - assert seq_lens_cpu is not None - max_seq_len = ( - max_seq_len_override - if max_seq_len_override is not None - else int(seq_lens_cpu.max().item()) - ) + if max_seq_len_override is not None: + max_seq_len = max_seq_len_override + elif seq_lens_cpu is not None: + max_seq_len = int(seq_lens_cpu.max().item()) + else: + # an eager DSpark verify under the overlap scheduler has device lengths only + max_seq_len = ( + getattr( + forward_batch.spec_info, "candidate_max_seq_len_upper_bound", None + ) + or self.MAX_SEQ_LEN_FOR_CAPTURE + ) if forward_batch.forward_mode.is_decode_or_idle(): # DSv4 bakes this step's KV write target (c4/c128) into metadata, @@ -1334,6 +1905,17 @@ class DeepseekV4HipRadixBackend( seq_lens=seq_lens, out_cache_loc=out_cache_loc, ) + elif ( + forward_batch.forward_mode.is_target_verify() + and self._uses_dspark_draft_window() + ): + metadata = self.init_forward_metadata_dspark_draft_block( + max_seq_len=max_seq_len, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + out_cache_loc=forward_batch.out_cache_loc, + block_size=int(forward_batch.spec_info.draft_token_num), + ) elif forward_batch.forward_mode.is_target_verify(): ragged_layout = resolve_ragged_verify_layout(forward_batch) metadata = self.init_forward_metadata_target_verify( @@ -1347,31 +1929,25 @@ class DeepseekV4HipRadixBackend( ), ragged_layout=ragged_layout, ) - elif forward_batch.forward_mode.is_prefill(include_draft_extend_v2=True): - extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu - extend_seq_lens = forward_batch.extend_seq_lens - assert ( - seq_lens is not None - and seq_lens_cpu is not None - and extend_seq_lens is not None - and extend_seq_lens_cpu is not None - ) - is_draft = ( - forward_batch.forward_mode.is_draft_extend_v2() or self.is_draft_worker - ) - metadata = self.init_forward_metadata_prefill( + elif forward_batch.forward_mode.is_draft_extend_v2(): + metadata = self.init_forward_metadata_draft_extend( max_seq_len=max_seq_len, req_pool_indices=req_pool_indices, seq_lens=seq_lens, - seq_lens_cpu=seq_lens_cpu.tolist(), + seq_lens_cpu=( + seq_lens_cpu.tolist() if seq_lens_cpu is not None else None + ), + num_tokens_per_req=self.speculative_num_draft_tokens, out_cache_loc=forward_batch.out_cache_loc, - num_tokens=sum(extend_seq_lens_cpu), - extend_seq_lens=extend_seq_lens, - extend_seq_lens_cpu=extend_seq_lens_cpu, - extend_start_loc=forward_batch.extend_start_loc, - need_compress=not is_draft, + ) + elif forward_batch.forward_mode.is_prefill(): + metadata = self._init_forward_metadata_prefill_from_batch( + forward_batch, + max_seq_len=max_seq_len, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu, use_prefill_cuda_graph=use_prefill_cuda_graph, - exact_num_tokens=is_draft, ) else: raise NotImplementedError(f"unsupported mode {forward_batch.forward_mode=}") @@ -1385,19 +1961,252 @@ class DeepseekV4HipRadixBackend( self.forward_metadata = self._build_forward_metadata(forward_batch) self.init_forward_metadata_in_graph(forward_batch) self._refresh_fp4_prefill_workspace(forward_batch) + self.tail_forward_metadata = ( + self._build_late_layer_tail_metadata(forward_batch) + if self.enable_decoder_swa_bounded_replay + and not self.is_draft_worker + and forward_batch.forward_mode.is_extend_without_speculative() + else None + ) + + def _init_forward_metadata_prefill_from_batch( + self, + forward_batch: ForwardBatch, + *, + max_seq_len: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: torch.Tensor, + use_prefill_cuda_graph: bool = False, + ) -> DSV4Metadata: + extend_seq_lens_cpu = forward_batch.extend_seq_lens_cpu + extend_seq_lens = forward_batch.extend_seq_lens + assert ( + seq_lens is not None + and seq_lens_cpu is not None + and extend_seq_lens is not None + and extend_seq_lens_cpu is not None + ) + is_draft = ( + forward_batch.forward_mode.is_draft_extend_v2() or self.is_draft_worker + ) + return self.init_forward_metadata_prefill( + max_seq_len=max_seq_len, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens_cpu.tolist(), + out_cache_loc=forward_batch.out_cache_loc, + num_tokens=sum(extend_seq_lens_cpu), + extend_seq_lens=extend_seq_lens, + extend_seq_lens_cpu=extend_seq_lens_cpu, + extend_start_loc=forward_batch.extend_start_loc, + need_compress=not is_draft, + use_prefill_cuda_graph=use_prefill_cuda_graph, + exact_num_tokens=is_draft, + ) + + # ---- decoder SWA bounded replay --------------------------------------- + + def _build_late_layer_tail_metadata( + self, forward_batch: ForwardBatch + ) -> DSV4Metadata: + """Metadata for the layers after the last kv_source layer under decoder SWA bounded + replay: each request's last SWA_WINDOW extend tokens, whose window is floored at the + tail start since no window KV before it is written at those layers.""" + extend_lens_cpu = forward_batch.extend_seq_lens_cpu + seq_lens_cpu = forward_batch.seq_lens_cpu + assert extend_lens_cpu is not None and seq_lens_cpu is not None + device = forward_batch.out_cache_loc.device + token_indices, tail_lens_cpu, swa_replay_start = late_layer_tail_layout( + extend_lens_cpu=extend_lens_cpu, + seq_lens_cpu=seq_lens_cpu.tolist(), + tail_len=SWA_WINDOW, + device=device, + ) + contiguous_start = ( + extend_lens_cpu[0] - tail_lens_cpu[0] if len(extend_lens_cpu) == 1 else None + ) + out_cache_loc = _tail_rows( + forward_batch.out_cache_loc, + token_indices=token_indices, + contiguous_start=contiguous_start, + ) + tail_lens = torch.tensor(tail_lens_cpu, dtype=torch.int32, device=device) + num_tokens = sum(tail_lens_cpu) + metadata = self.init_forward_metadata_prefill( + max_seq_len=int(seq_lens_cpu.max().item()), + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens.to(torch.int32), + seq_lens_cpu=seq_lens_cpu.tolist(), + out_cache_loc=out_cache_loc, + num_tokens=num_tokens, + extend_seq_lens=tail_lens, + extend_seq_lens_cpu=tail_lens_cpu, + extend_start_loc=torch.cumsum(tail_lens, dim=0) - tail_lens, + swa_replay_start=swa_replay_start, + ) + request_layout = metadata.core_attn_metadata.request_window_layout + swa_out_cache_loc = ( + request_layout.write_loc + if request_layout is not None + else self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc).to( + torch.int32 + ) + ) + metadata.core_attn_metadata.swa_out_cache_loc = swa_out_cache_loc + metadata.low_ratio_req_indices = torch.repeat_interleave( + forward_batch.req_pool_indices.to(torch.int64), + tail_lens.to(torch.int64), + output_size=num_tokens, + ) + positions = _tail_rows( + forward_batch.positions, + token_indices=token_indices, + contiguous_start=contiguous_start, + ) + metadata.low_ratio_pos_i64 = positions.to(torch.int64) + metadata.late_layer_tail = LateLayerTail( + token_indices=token_indices, + positions=positions, + extend_seq_lens=tail_lens, + extend_seq_lens_cpu=tail_lens_cpu, + swa_out_cache_loc=swa_out_cache_loc, + contiguous_start=contiguous_start, + ) + self._refresh_fp4_prefill_workspace(forward_batch, metadata) + return metadata + + def enter_late_layer_tail(self, forward_batch: ForwardBatch) -> tuple: + """Switch the late layers onto the tail metadata; exit_late_layer_tail takes the + return value. Each request's candidate mask is cut to its tail rows.""" + tail_metadata = self.tail_forward_metadata + assert tail_metadata is not None, "no tail metadata for this forward" + saved = (self.forward_metadata, self.candidate_masks) + tail = tail_metadata.late_layer_tail + tail_lens_cpu = tail.extend_seq_lens_cpu + if isinstance(self.candidate_masks, list) and self.candidate_masks: + # the last tail_len rows of each request's candidate publication + self.candidate_masks = [ + ( + None + if mask is None + else ( + slice_candidate_blocks( + mask, slice(mask.ids.shape[0] - t, mask.ids.shape[0]) + ) + if isinstance(mask, CandidateBlocks) + else mask[mask.shape[0] - t :] + ) + ) + for mask, t in zip(self.candidate_masks, tail_lens_cpu) + ] + # the consumers after the switch read the tail buffers, so carry the last source's rows over + full_core = saved[0].core_attn_metadata + tail_core = tail_metadata.core_attn_metadata + for ratio in tail_core.low_ratios: + for full_buf, tail_buf in ( + ( + full_core.sparse_page_indices(ratio), + tail_core.sparse_page_indices(ratio), + ), + ( + full_core.sparse_topk_lengths(ratio), + tail_core.sparse_topk_lengths(ratio), + ), + ( + full_core.sparse_raw_indices(ratio), + tail_core.sparse_raw_indices(ratio), + ), + ): + if full_buf is None or tail_buf is None: + continue + rows = tail.real_rows(full_buf) + tail_buf[: rows.shape[0]].copy_(rows) + if tail.pad_rows: + tail_buf[rows.shape[0] :].fill_(0 if tail_buf.ndim == 1 else -1) + self.forward_metadata = tail_metadata + window = self.token_to_kv_pool.request_window + if window is not None: + window.activate(tail_core.request_window_layout) + return saved + + def exit_late_layer_tail(self, saved: tuple, forward_batch: ForwardBatch) -> None: + self.forward_metadata, self.candidate_masks = saved + window = self.token_to_kv_pool.request_window + if window is not None: + window.activate( + self.forward_metadata.core_attn_metadata.request_window_layout + ) + + # ---- breakable prefill CUDA graphs ----------------------------------- + # only the fused KV store is recorded in a segment, and it reads the SWA store target by address + + def _prefill_metadata_for_batch( + self, + forward_batch: ForwardBatch, + *, + max_seq_len_override: Optional[int] = None, + ) -> DSV4Metadata: + """The eager prefill build, for a capture batch or a live replay batch.""" + seq_lens_cpu = forward_batch.seq_lens_cpu + assert seq_lens_cpu is not None + actual_max_seq_len = int(seq_lens_cpu.max().item()) + if max_seq_len_override is None: + max_seq_len_override = forward_batch.max_seq_len_override + max_seq_len = ( + actual_max_seq_len if max_seq_len_override is None else max_seq_len_override + ) + if actual_max_seq_len > max_seq_len: + raise ValueError( + "Prefill CUDA graph max context size is smaller than the " + f"live context: {max_seq_len=} < {actual_max_seq_len=}" + ) + return self._init_forward_metadata_prefill_from_batch( + forward_batch, + max_seq_len=max_seq_len, + req_pool_indices=forward_batch.req_pool_indices, + seq_lens=forward_batch.seq_lens.to(torch.int32), + seq_lens_cpu=seq_lens_cpu, + ) def init_forward_metadata_for_breakable_cuda_graph_capture( self, forward_batch: ForwardBatch - ): - self.forward_metadata = self._build_forward_metadata( - forward_batch, - max_seq_len_override=self.MAX_SEQ_LEN_FOR_CAPTURE, - use_prefill_cuda_graph=True, + ) -> DSV4Metadata: + if not self.low_ratios: + self.forward_metadata = self._build_forward_metadata( + forward_batch, + max_seq_len_override=self.MAX_SEQ_LEN_FOR_CAPTURE, + use_prefill_cuda_graph=True, + ) + self.init_forward_metadata_in_graph(forward_batch) + self._refresh_fp4_prefill_workspace(forward_batch) + assert isinstance(self.forward_metadata, DSV4Metadata) + return self.forward_metadata + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + is_unified_kv_triton, ) + + if is_unified_kv_triton(): + raise NotImplementedError( + "breakable prefill CUDA graphs on HIP need the flash_mla attention " + "path: the unified_kv prefill runs attention without the graph break" + ) + if self.has_c4 or self.has_c128: + raise NotImplementedError( + "breakable prefill CUDA graphs on HIP support only the DeepSeek-V4.1 layout " + "(ratios 1 / 2); the c4 / c128 metadata is not pinned for the captured segments" + ) + if get_parallel().attn_cp_size != 1: + raise NotImplementedError( + "breakable prefill CUDA graphs on HIP do not support attention CP" + ) + assert forward_batch.forward_mode.is_extend(), forward_batch.forward_mode + metadata = self._prefill_metadata_for_batch(forward_batch) + self.forward_metadata = metadata + # the captured store kernels bind this target's address self.init_forward_metadata_in_graph(forward_batch) self._refresh_fp4_prefill_workspace(forward_batch) - assert isinstance(self.forward_metadata, DSV4Metadata) - return self.forward_metadata + return metadata def prepare_forward_metadata_for_breakable_cuda_graph_replay( self, @@ -1406,24 +2215,55 @@ class DeepseekV4HipRadixBackend( *, static_forward_batch: Optional[ForwardBatch] = None, ) -> None: - replay_batch = ( - static_forward_batch if static_forward_batch is not None else forward_batch - ) - replay_metadata = self._build_forward_metadata( - replay_batch, - max_seq_len_override=self.MAX_SEQ_LEN_FOR_CAPTURE, - use_prefill_cuda_graph=True, - ) - self.forward_metadata = replay_metadata - self.init_forward_metadata_in_graph(replay_batch) + if not self.low_ratios: + replay_batch = ( + static_forward_batch + if static_forward_batch is not None + else forward_batch + ) + replay_metadata = self._build_forward_metadata( + replay_batch, + max_seq_len_override=self.MAX_SEQ_LEN_FOR_CAPTURE, + use_prefill_cuda_graph=True, + ) + self.forward_metadata = replay_metadata + self.init_forward_metadata_in_graph(replay_batch) - assert isinstance(capture_metadata, DSV4Metadata) - assert isinstance(replay_metadata, DSV4Metadata) - capture_metadata.refresh_for_breakable_cuda_graph_replay_(replay_metadata) + assert isinstance(capture_metadata, DSV4Metadata) + assert isinstance(replay_metadata, DSV4Metadata) + capture_metadata.refresh_for_breakable_cuda_graph_replay_(replay_metadata) + self.forward_metadata = capture_metadata + self._refresh_fp4_prefill_workspace(replay_batch) + return + assert isinstance(capture_metadata, DSV4Metadata), type(capture_metadata) + if static_forward_batch is None: + static_forward_batch = forward_batch + # break-time consumers read the live batch's eager build; the padded loc feeds only the target + live = self._prefill_metadata_for_batch( + forward_batch, + max_seq_len_override=static_forward_batch.max_seq_len_override, + ) + self.forward_metadata = live + self.init_forward_metadata_in_graph(static_forward_batch) + self._refresh_fp4_prefill_workspace(forward_batch) + capture_metadata.refresh_for_breakable_cuda_graph_replay_(live) self.forward_metadata = capture_metadata - self._refresh_fp4_prefill_workspace(replay_batch) + + @property + def verify_mask(self) -> Optional[VerifyMask]: + return self._verify_mask def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int) -> None: + # verify metadata never reads the tree mask, so EAGLE builds it QLEN_ONLY + self._verify_mask = maybe_create_verify_mask( + is_draft_runner=self.is_draft_worker, + skip_prefill=False, + max_bs=max_bs, + max_context_len=self.max_context_len, + num_draft_tokens=self.speculative_num_draft_tokens, + device=self.device, + is_read=False, + ) self.cuda_graph_metadata_of_bucket_and_bs: Dict[ _GraphBucket, Dict[ @@ -1470,6 +2310,10 @@ class DeepseekV4HipRadixBackend( core.c0_flashmla_metadata = _create_flashmla_metadata() core.c4_flashmla_metadata = _create_flashmla_metadata() core.c128_flashmla_metadata = _create_flashmla_metadata() + if 1 in core.low_ratios: + core.c1_flashmla_metadata = _create_flashmla_metadata() + if 2 in core.low_ratios: + core.c2_flashmla_metadata = _create_flashmla_metadata() # Warmup upgraded raw->full on the host; # restore raw so capture re-runs the upgrade inside the graph. @@ -1877,6 +2721,15 @@ class DeepseekV4HipRadixBackend( def get_swa_out_cache_loc(self, forward_batch: ForwardBatch) -> torch.Tensor: # Idle metadata may be stale; zero-padded locations target the dummy slot. + window = self.token_to_kv_pool.request_window + if window is not None: + layout = self.forward_metadata.core_attn_metadata.request_window_layout + window.activate(layout) + return layout.write_loc + metadata = self.forward_metadata + if isinstance(metadata, DSV4Metadata) and metadata.late_layer_tail is not None: + # the tail's rows are a subset of the extend, so it owns its own store target + return metadata.late_layer_tail.swa_out_cache_loc out_cache_loc = forward_batch.out_cache_loc core = getattr(self.forward_metadata, "core_attn_metadata", None) cached = core.swa_out_cache_loc if core is not None else None @@ -1929,20 +2782,23 @@ class DeepseekV4HipRadixBackend( cache_k=swa_k, ) - def forward( + def _forward_attention( self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, layer: RadixAttention, forward_batch: ForwardBatch, - compress_ratio: Literal[0, 4, 128], + compress_ratio: Literal[0, 1, 2, 4, 128], save_kv_cache: bool = True, attn_sink: Optional[torch.Tensor] = None, q_rope: Optional[torch.Tensor] = None, k_rope: Optional[torch.Tensor] = None, + inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, **_, ) -> torch.Tensor: + """inv_rope=(freqs_real, positions) has the kernel apply the inverse RoPE + to the last freqs_real.shape[-1] dims of every output head.""" if self.mtp_enabled and forward_batch.forward_mode.is_idle(): return q.new_empty(q.shape[0], q.shape[1], layer.v_head_dim) @@ -1960,7 +2816,7 @@ class DeepseekV4HipRadixBackend( ) if is_unified_kv_triton(): - return self._forward_unified_kv( + o = self._forward_unified_kv( q=q, kv=swa_k, layer=layer, @@ -1972,6 +2828,10 @@ class DeepseekV4HipRadixBackend( q_rope=q_rope, k_rope=k_rope, ) + assert inv_rope is None, ( + "the unified_kv model path applies its inverse RoPE" + ) + return o if isinstance(core_attn_metadata, DSV4AttnMetadata): if save_kv_cache: @@ -1979,10 +2839,12 @@ class DeepseekV4HipRadixBackend( swa_k_cache = token_to_kv_pool.get_swa_key_buffer_radix(layer_id) extra_k_cache, extra_indices, extra_topk_lengths = None, None, None - if compress_ratio == 4: + if compress_ratio in (1, 2, 4): extra_k_cache = token_to_kv_pool.get_extra_key_buffer(layer_id) - extra_indices = core_attn_metadata.c4_sparse_page_indices - extra_topk_lengths = core_attn_metadata.c4_sparse_topk_lengths + extra_indices = core_attn_metadata.sparse_page_indices(compress_ratio) + extra_topk_lengths = core_attn_metadata.sparse_topk_lengths( + compress_ratio + ) elif compress_ratio == 128: extra_k_cache = token_to_kv_pool.get_extra_key_buffer(layer_id) extra_indices = core_attn_metadata.c128_page_indices @@ -1990,20 +2852,28 @@ class DeepseekV4HipRadixBackend( swa_page_size = token_to_kv_pool.swa_page_size assert swa_k_cache.ndim == 2 - k_cache_total_dim = token_to_kv_pool.swa_kv_pool.kv_cache_total_dim + swa_pool = ( + token_to_kv_pool.request_window.state + if token_to_kv_pool.request_window is not None + else token_to_kv_pool.swa_kv_pool + ) + k_cache_total_dim = swa_pool.kv_cache_total_dim swa_k_cache = swa_k_cache[:, : swa_page_size * k_cache_total_dim].view( swa_k_cache.shape[0], swa_page_size, 1, k_cache_total_dim ) if extra_k_cache is not None: extra_page_size = token_to_kv_pool.get_extra_key_page_size(layer_id) + extra_cache_dim = token_to_kv_pool.get_extra_key_bytes_per_token( + layer_id + ) extra_k_cache = extra_k_cache[ - :, : extra_page_size * k_cache_total_dim + :, : extra_page_size * extra_cache_dim ].view( extra_k_cache.shape[0], extra_page_size, 1, - k_cache_total_dim, + extra_cache_dim, ) swa_page_indices = core_attn_metadata.swa_page_indices swa_topk_lengths = core_attn_metadata.swa_topk_lengths @@ -2034,11 +2904,50 @@ class DeepseekV4HipRadixBackend( f"{extra_indices.shape=}'s last dimension is not aligned to 64" ) - from sglang.srt.layers.attention.hip_flash_mla import ( - flash_mla_with_kvcache_entrypoint, - ) - - backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get() + backend = resolve_hip_flashmla_backend() + if ( + backend == "aiter_sparse" + and 0 < q.shape[0] <= SWAPAB_MAX_BATCH + and q.shape[1:] == (1, SWAPAB_NUM_HEADS, SWAPAB_HEAD_DIM) + and q.dtype == torch.bfloat16 + and self.head_dim_v == SWAPAB_HEAD_DIM + and self.softmax_scale == SWAPAB_SOFTMAX_SCALE + and swa_k_cache.shape[-1] == KVLayout.V4.bytes_per_token + and ( + extra_k_cache is None + or extra_k_cache.shape[-1] == KVLayout.V4.bytes_per_token + ) + and hip_attn_kv_splits() == 0 + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + or forward_batch.forward_mode.is_draft_extend_v2() + ) + ): + return swapab_attention( + q, + swa_k_cache, + swa_page_indices, + swa_topk_lengths, + attn_sink, + extra_k_cache, + extra_indices, + extra_topk_lengths, + inv_rope=inv_rope, + ) + # the V4.1 compact kernels read topk_length themselves + if ( + backend == "aiter_sparse" + and swa_k_cache.shape[-1] == KVLayout.V4.bytes_per_token + ): + swa_page_indices, extra_indices = _fold_lengths_for_aiter_sparse( + core_attn_metadata, + compress_ratio, + swa_page_indices, + swa_topk_lengths, + extra_indices, + extra_topk_lengths, + ) input_dict = dict( q=q, k_cache=swa_k_cache, @@ -2055,6 +2964,8 @@ class DeepseekV4HipRadixBackend( extra_indices_in_kvcache=extra_indices, extra_topk_length=extra_topk_lengths, ) + if inv_rope is not None: + input_dict["inv_rope"] = inv_rope o = flash_mla_with_kvcache_entrypoint(**input_dict, backend=backend)[0] o = o.squeeze(1) @@ -2062,6 +2973,48 @@ class DeepseekV4HipRadixBackend( raise NotImplementedError("ragged attention") + def _low_ratio_index_topk( + self, + layer, + x, + q_lora, + req, + pos, + forward_batch, + *, + rows_per_request=None, # NOTE: only used in CP + ) -> None: + """FlyDSL fp4 paged logits for decode, target-verify and ragged prefill; + target-verify takes the decode body (one-token rows).""" + # every body below rewrites the ratio's page indices in place + self.forward_metadata.core_metadata.drop_folded_sparse_indices( + layer.compress_ratio + ) + if ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ): + low_ratio_index_topk_hip_decode( + self, layer, x, q_lora, pos, forward_batch=forward_batch + ) + elif forward_batch.forward_mode.is_extend(): + if envs.SGLANG_DSV41_TORCH_PREFILL_INDEXER.get(): + raise NotImplementedError( + "SGLANG_DSV41_TORCH_PREFILL_INDEXER (the torch prefill indexer " + "oracle) is not supported on HIP" + ) + # HIP rejects prefill CP, the only caller passing rows_per_request + assert rows_per_request is None, "prefill CP is not supported on HIP" + assert ( + forward_batch.seq_lens_cpu is not None + and forward_batch.extend_seq_lens_cpu is not None + ), "the HIP low-ratio prefill indexer needs the batch's CPU lengths" + low_ratio_index_topk_hip_extend(self, layer, x, q_lora, pos, forward_batch) + else: + raise NotImplementedError( + f"low-ratio indexer for {forward_batch.forward_mode} on HIP" + ) + def expand_extend_with_same_length( self, bs: int, @@ -2088,25 +3041,81 @@ class DeepseekV4HipRadixBackend( out_loc: torch.Tensor, need_compress: bool = True, is_prefill: bool = False, + dspark_block_size: Optional[int] = None, + swa_replay_start: Optional[torch.Tensor] = None, + num_groups: Optional[int] = None, ) -> DSV4AttnMetadata: + """swa_replay_start floors every row's window at that absolute position.""" seq_lens_casual = seq_lens_casual.to(torch.int32) - - swa_page_indices = self.get_swa_page_indices( - seq_lens_casual=seq_lens_casual, - req_pool_indices_repeated=req_pool_indices_repeated, - ) - - swa_page_indices = _pad_last_dim( - swa_page_indices, multiples_of=PAGE_INDEX_ALIGNED_SIZE - ) - raw_positions = seq_lens_casual - 1 - swa_topk_lengths = torch.clamp(seq_lens_casual, max=SWA_WINDOW) - page_table = req_to_token[ - req_pool_indices_repeated, : max_seq_len : self.page_size - ] - page_table = (page_table // self.page_size).to(torch.int32) + request_layout = None + window = self.token_to_kv_pool.request_window + if window is not None: + if self.encoder_replay: + starts = torch.ones_like(raw_positions, dtype=torch.bool) + starts[1:] = ( + req_pool_indices_repeated[1:] != req_pool_indices_repeated[:-1] + ) + offset = torch.arange( + raw_positions.numel(), device=raw_positions.device + ) + group_first = torch.cummax(torch.where(starts, offset, 0), dim=0).values + swa_replay_start = raw_positions - (offset - group_first) + request_layout = window_layout( + req_pool_indices_repeated, + raw_positions, + capacity=window.capacity, + floor=swa_replay_start, + num_groups=num_groups, + ) + swa_page_indices = _pad_last_dim( + request_layout.indices, multiples_of=PAGE_INDEX_ALIGNED_SIZE + ) + swa_topk_lengths = request_layout.lengths + elif dspark_block_size is not None: + assert self._uses_dspark_draft_window() and ( + dspark_block_size == self.target_verify_num_draft_tokens + ), ( + f"{dspark_block_size=} must equal target_verify_num_draft_tokens=" + f"{self.target_verify_num_draft_tokens} on the DSpark draft backend" + ) + # Already padded to PAGE_INDEX_ALIGNED_SIZE by the builder. + swa_page_indices, swa_topk_lengths = self.get_dspark_swa_page_indices( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_pool_indices_repeated, + out_loc=out_loc, + block_size=dspark_block_size, + ) + elif swa_replay_start is None: + swa_page_indices = self.get_swa_page_indices( + seq_lens_casual=seq_lens_casual, + req_pool_indices_repeated=req_pool_indices_repeated, + ) + swa_page_indices = _pad_last_dim( + swa_page_indices, multiples_of=PAGE_INDEX_ALIGNED_SIZE + ) + swa_topk_lengths = torch.clamp(seq_lens_casual, max=SWA_WINDOW) + else: + swa_page_indices = BuildCausalSwaPageIndices.execute( + req_to_token=self.req_to_token, + full_to_swa_mapping=self.token_to_kv_pool.full_to_swa_index_mapping, + req_pool_indices_repeated=req_pool_indices_repeated, + seq_lens_casual=seq_lens_casual, + swa_window=SWA_WINDOW, + page_index_aligned_size=PAGE_INDEX_ALIGNED_SIZE, + swa_replay_start=swa_replay_start, + ) + # Slots below the floor are -1; the valid count shrinks to match. + floored = raw_positions - swa_replay_start.to(raw_positions.dtype) + 1 + swa_topk_lengths = torch.minimum( + torch.clamp(seq_lens_casual, max=SWA_WINDOW), floored.clamp_min(0) + ) + + # the gather, the floor division and the cast in one launch + page_table = page_table_from_req_to_token( + req_to_token, req_pool_indices_repeated, max_seq_len, self.page_size + ) core_attn_metadata = DSV4AttnMetadata( page_size=self.page_size, @@ -2118,6 +3127,8 @@ class DeepseekV4HipRadixBackend( swa_page_indices=swa_page_indices, swa_topk_lengths=swa_topk_lengths, index_topk=self.index_topk, + low_ratios=self.low_ratios, + request_window_layout=request_layout, ) if need_compress: @@ -2154,6 +3165,8 @@ class DeepseekV4HipRadixBackend( # flash_mla attention requires int32 page indices. return swa_indices.to(torch.int32) + get_dspark_swa_page_indices = DeepseekV4AttnBackend.get_dspark_swa_page_indices + class DeepseekV4MultiStepBackend(DeepseekV4HipRadixBackend): def __init__( diff --git a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py index 2ffce44b4a..7a32a5a107 100644 --- a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py +++ b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py @@ -16,19 +16,31 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import ( fake_quant_fp4, ) from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32 from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.utils import add_prefix +from sglang.srt.utils import add_prefix, is_gfx95_supported -def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False): - if x.is_cuda and torch.version.cuda is not None and x.dtype == torch.bfloat16: +def _fused_dsv41_kernels(x: torch.Tensor) -> bool: + # validated on CUDA and gfx950; other ROCm GPUs keep the torch forms + return x.is_cuda and (torch.version.cuda is not None or is_gfx95_supported()) + + +def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False, positions=None): + """RoPE plus fake FP4 quantization, fused for bf16 inputs on CUDA and ROCm. With + positions, freqs is the whole table and the fused kernel gathers freqs[positions].""" + if _fused_dsv41_kernels(x) and x.dtype == torch.bfloat16: from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( rope_tail_fake_quant_fp4, ) - return rope_tail_fake_quant_fp4(x, freqs, rope_dim, compressed_kv=compressed_kv) + return rope_tail_fake_quant_fp4( + x, freqs, rope_dim, compressed_kv=compressed_kv, positions=positions + ) + if positions is not None: + freqs = freqs[positions] quant = fake_quant_compressed_kv if compressed_kv else fake_quant_fp4 return quant(rope_tail(x, freqs, rope_dim)) @@ -42,9 +54,9 @@ class RMSNorm(nn.Module): self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: + # the Triton kernel is bitwise this fallback for bf16 rows on CUDA and ROCm (gfx950) if ( - x.is_cuda - and torch.version.cuda is not None + _fused_dsv41_kernels(x) and x.dtype in (torch.bfloat16, torch.float32) and self.weight.dtype in (torch.bfloat16, torch.float32) and x.shape[-1] in (128, 512) @@ -87,11 +99,13 @@ def rope_tail( def fused_low_ratio_compress_supported() -> bool: - """The fused c1 / c2 / index-K decode kernels pack fp4 with - `cvt.rn.satfinite.e2m1x2`, an sm100+ instruction; the answer also fixes the + """The fused c1 / c2 / index-K decode kernels need sm100+ on CUDA (they pack fp4 + with `cvt.rn.satfinite.e2m1x2`) or gfx950 on ROCm; the answer also fixes the ratio-2 weight layout (`wkv_gate`, or `wkv` plus `wgate`).""" - if not torch.cuda.is_available() or torch.version.hip is not None: + if not torch.cuda.is_available(): return False + if torch.version.hip is not None: + return is_gfx95_supported() return torch.cuda.get_device_capability()[0] >= 10 @@ -208,6 +222,10 @@ class DeepseekV41Indexer(nn.Module): quant_config=None, prefix=add_prefix("weights_proj", prefix), ) + # -1 off gfx950 or for shapes the split-K GEMV lacks + self.weights_proj_hip_max_tokens = rocm_gemv_split_k_max_tokens( + n=self.n_heads, k=config.hidden_size, weight_dtype=torch.bfloat16 + ) # The decode GEMM matches tiny_gemm's reduction order, not cuBLAS's. self.weights_proj_small_max_m = _small_weights_proj_max_m( self.n_heads, config.hidden_size @@ -236,10 +254,17 @@ class DeepseekV41Indexer(nn.Module): k = self.k_norm(self.forward_wk(latent)) return _rope_fq4(k, freqs, self.rope_head_dim) - def queries(self, q_lora: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + def queries( + self, + q_lora: torch.Tensor, + freqs: torch.Tensor, + positions: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """freqs per token, or the whole table with positions (gathered inside the + fused RoPE launch).""" q, _ = self.wq_b(q_lora) q = q.view(q.shape[0], self.n_local_heads, self.index_head_dim) - return _rope_fq4(q, freqs, self.rope_head_dim) + return _rope_fq4(q, freqs, self.rope_head_dim, positions=positions) def head_weights_raw(self, x: torch.Tensor) -> torch.Tensor: """`weights_proj(x)` before the scale, [tokens, n_heads] bf16.""" diff --git a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py new file mode 100644 index 0000000000..b3b3efa110 --- /dev/null +++ b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py @@ -0,0 +1,611 @@ +"""DeepSeek V4.1 low-ratio (1 / 2) indexer on ROCm: FlyDSL fp4 paged MQA logits over the split +payload / scale index-K pools, then the AOT top-k transform -- the DeepGEMM path's contract.""" + +from __future__ import annotations + +import itertools +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple + +import torch +import triton + +from sglang.kernels.ops.attention.dsv4 import topk_transform_paged +from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( + CandidateBlocks, + cat_candidate_blocks, + select_candidate_blocks_hip, + slice_candidate_blocks, + topk_transform_paged_sorted, + topk_within_candidate_blocks_hip, +) +from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + LOW_RATIO_PAGE_TABLE_BUCKET, + FP4DecodeWorkspace, + FP4PrefillWorkspace, + aiter_fp4_paged_mqa_logits, + index_q_rope_pack_weights_flydsl, + indexer_head_weights, + logits_rows_per_chunk, + pack_fp4_query_flydsl, + prepare_fp4_decode_workspace, + prepare_fp4_prefill_workspace, +) +from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( + index_k_norm_rope_pack_store_split, +) +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.srt.layers.attention.deepseek_v4_backend import ( + _as_int_list, +) + +if TYPE_CHECKING: + from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +# fp32 logits scored per candidate-block chunk stay under this many bytes +_CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30 + + +def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool: + """Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch.""" + return ( + 0 < x.shape[0] <= indexer.weights_proj_hip_max_tokens + and x.dim() == 2 + and x.dtype == torch.bfloat16 + and x.stride(1) == 1 + ) + + +def _indexer_head_weights(indexer, x: torch.Tensor) -> torch.Tensor: + """indexer.head_weights(x) as the contiguous bf16 [T, H] the FlyDSL kernels take; decode row + counts run indexer_head_weights (same two roundings as aiter's GEMM plus the scale).""" + if _gemv_head_weight_rows(indexer, x): + return indexer_head_weights( + x, indexer.weights_proj.weight, indexer.head_weight_scale + ) + return indexer.head_weights(x).contiguous() + + +def _indexer_inputs(layer, x, q_lora, pos): + """(payload, scale) query in the FlyDSL layout and the pre-scaled head weights.""" + indexer = layer.indexer + # The kernel sums head scores locally, so the indexer heads must be replicated. + assert indexer.n_local_heads == indexer.n_heads + if ( + _gemv_head_weight_rows(indexer, x) + and indexer.n_heads % 16 == 0 + and indexer.n_heads <= 64 + and indexer.index_head_dim == 128 + and layer.freqs_cis.dtype == torch.complex64 + ): + # decode rows: wq_b, split-K head-weight GEMV, then one launch for RoPE, fp4 pack and reduce + q, _ = indexer.wq_b(q_lora) + partials = rocm_router_gemv_split_k(x, indexer.weights_proj.weight) + return index_q_rope_pack_weights_flydsl( + q, + layer.freqs_cis, + pos, + indexer.rope_head_dim, + partials, + indexer.head_weight_scale, + num_heads=indexer.n_heads, + ) + # [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself + q = indexer.queries(q_lora, layer.freqs_cis, positions=pos) + q_fp4, q_scale = pack_fp4_query_flydsl(q) + weights = _indexer_head_weights(indexer, x) # [T, H] bf16, already scaled + return q_fp4, q_scale, weights + + +def build_low_ratio_decode_workspaces( + metadata_by_ratio: Dict[int, PagedIndexerMetadata], +) -> Dict[int, FP4DecodeWorkspace]: + """Capture-safe: everything the schedule kernel touches is pinned in the workspace.""" + return { + ratio: prepare_fp4_decode_workspace( + meta.page_table, + meta.compressed_seq_lens, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + ) + for ratio, meta in metadata_by_ratio.items() + } + + +def refresh_low_ratio_prefill_workspaces( + metadata_by_ratio: Dict[int, PagedIndexerMetadata], + previous: Optional[Dict[int, FP4PrefillWorkspace]], +) -> Dict[int, FP4PrefillWorkspace]: + """Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace.""" + previous = previous or {} + return { + ratio: prepare_fp4_prefill_workspace( + meta.page_table, + meta.compressed_seq_lens, + workspace=previous.get(ratio), + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + ) + for ratio, meta in metadata_by_ratio.items() + } + + +def low_ratio_identity_skip_enabled( + *, index_topk: int, candidate_topk_blocks: int, candidate_block_size: int +) -> bool: + """Whether identity requests may bypass scoring under this model config: the score-free + candidate mask is right only while <= index_topk positions fit the candidate top-k.""" + return triton.cdiv(index_topk, candidate_block_size) <= candidate_topk_blocks + + +def _fill_identity_request( + *, + lc: int, + slots_j: torch.Tensor, + lens: torch.Tensor, + page_rows: torch.Tensor, + raw_rows: Optional[torch.Tensor], +) -> None: + """Rows [t, topk] of the -1 filled buffers: column j < lc holds compressed + position j (its slot / its raw index) when j < lens_row, else -1. An identity + request publishes no candidates: every reachable block is one.""" + j = torch.arange(lc, device=lens.device) + reach = j[None, :] < lens[:, None] + page_rows[:, :lc] = torch.where(reach, slots_j[None, :], -1).to(torch.int32) + if raw_rows is not None: + raw_rows[:, :lc] = torch.where(reach, j[None, :], -1).to(torch.int32) + + +def _decode_batch_max_seq_len(forward_batch) -> Optional[int]: + """The host-side longest context of a decode batch, None when unknown.""" + seq_lens_cpu = forward_batch.seq_lens_cpu + if seq_lens_cpu is None or seq_lens_cpu.numel() == 0: + return None + return int(seq_lens_cpu.max().item()) + + +def low_ratio_decode_rows_are_identity(backend, forward_batch, ratio: int) -> bool: + """Decode: every row's compressed context fits index_topk, so nothing needs scoring. Inside a + captured graph the answer is the variant being captured; eagerly it is the host-side batch + maximum. Target-verify rows keep the scored path.""" + if not backend.low_ratio_identity_skip or forward_batch is None: + return False + if not forward_batch.forward_mode.is_decode(): + return False + from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_is_capture_mode, + skip_low_ratio_indexer, + ) + + if get_is_capture_mode(): + return skip_low_ratio_indexer(ratio) + max_len = _decode_batch_max_seq_len(forward_batch) + return max_len is not None and max_len // ratio <= backend.index_topk + + +def low_ratio_candidate_span(hf_text_config) -> Optional[int]: + """The span (in positions) within which every block of a request is a + candidate, when the model has a candidate source; None otherwise.""" + # optional HF-config keys: a model without a candidate source lacks them + if getattr(hf_text_config, "candidate_source_layer_id", -1) < 0: + return None + span = getattr(hf_text_config, "candidate_topk_blocks", 0) * getattr( + hf_text_config, "candidate_block_size", 0 + ) + return span if span > 0 else None + + +def low_ratio_decode_rows_fit_candidate_span(backend, forward_batch) -> bool: + """Decode: every request's context fits the candidate span, so two-level top-k equals the plain + paged top-k. Captured: the variant being captured; eager: the host-side batch maximum.""" + span = backend.low_ratio_candidate_span + if span is None or forward_batch is None: + return False + if not ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ): + return False + from sglang.srt.model_executor.runner_utils.capture_mode import ( + get_capture_attention_variant, + get_is_capture_mode, + ) + + if get_is_capture_mode(): + return get_capture_attention_variant() in ( + "candidate_all", + "candidate_c2_all", + "candidate_unfiltered", + ) + max_len = _decode_batch_max_seq_len(forward_batch) + if forward_batch.forward_mode.is_target_verify(): + if max_len is None: + return False + max_len += backend.speculative_num_draft_tokens + return max_len is not None and max_len <= span + + +def low_ratio_index_topk_hip_decode( + backend, layer, x, q_lora, pos, forward_batch: ForwardBatch +) -> None: + """One token per request: paged fp4 logits over every visible compressed slot, level-one + candidate blocks where the layer publishes or consumes them, then the top-k transform + writes the -1 padded page indices.""" + pool = backend.token_to_kv_pool + metadata = backend.forward_metadata + core = metadata.core_metadata + ratio = layer.compress_ratio + indexer = layer.indexer + indexer_metadata = metadata.low_ratio_indexer_metadata(ratio) + assert indexer_metadata is not None, f"no decode indexer metadata for {ratio = }" + page_indices = core.sparse_page_indices(ratio) + raw_indices = core.sparse_raw_indices(ratio) + + if low_ratio_decode_rows_are_identity(backend, forward_batch, ratio): + # every row takes the transform's sequential branch (len <= topk), which reads no scores + scores = torch.empty( + (page_indices.shape[0], 1), dtype=torch.float32, device=pos.device + ) + topk_transform_paged( + scores, + indexer_metadata.compressed_seq_lens, + indexer_metadata.page_table, + page_indices, + indexer_metadata.compressed_page_size, + raw_indices, + ) + return + + two_level = indexer.is_candidate_source or indexer.uses_candidates + if two_level and low_ratio_decode_rows_fit_candidate_span(backend, forward_batch): + # every block is a candidate: the source publishes nothing, every layer runs the plain top-k + if indexer.is_candidate_source: + backend.candidate_masks = None + two_level = False + + q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos) + logits = aiter_fp4_paged_mqa_logits( + q_fp4=q_fp4, + q_scale=q_scale, + k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id), + k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id), + weights=weights, + page_table=indexer_metadata.page_table, + c4_seq_lens=indexer_metadata.compressed_seq_lens, + weight_scale=1.0, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + is_decode=True, + decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio), + ) + # level one is bounded on device by the compressed lengths: a captured step cannot read them back + if two_level and indexer.uses_candidates: + candidates = backend.candidate_masks + assert ( + isinstance(candidates, CandidateBlocks) + and candidates.ids.shape[0] == logits.shape[0] + and candidates.block_size == indexer.candidate_block_size + ), "candidate blocks missing for decode" + topk_within_candidate_blocks_hip( + logits, + indexer_metadata.compressed_seq_lens, + candidates, + page_table=indexer_metadata.page_table, + page_size=indexer_metadata.compressed_page_size, + page_indices=core.sparse_page_indices(ratio), + raw_indices=core.sparse_raw_indices(ratio), + sort_output=True, + ) + return + if two_level and indexer.is_candidate_source: + backend.candidate_masks = select_candidate_blocks_hip( + logits, + indexer_metadata.compressed_seq_lens, + topk_blocks=indexer.candidate_topk_blocks, + block_size=indexer.candidate_block_size, + ) + topk_transform_paged_sorted( + logits, + indexer_metadata.compressed_seq_lens, + indexer_metadata.page_table, + page_indices, + indexer_metadata.compressed_page_size, + raw_indices, + ) + + +def _extend_k_slots(req_to_token, *, ratio, lc_per_req, req_pool_indices, device): + """Per request, the c1/c2 pool slots of its visible compressed positions, and + each request's start offset in their concatenation.""" + starts = [0, *itertools.accumulate(lc_per_req)][:-1] + counts = torch.tensor(lc_per_req, dtype=torch.int64, device=device) + req_rows = torch.repeat_interleave(req_pool_indices.to(torch.int64), counts) + j = torch.arange(int(counts.sum()), device=device) - torch.repeat_interleave( + torch.tensor(starts, dtype=torch.int64, device=device), counts + ) + slots = req_to_token[req_rows, j * ratio].to(torch.int64) // ratio + return slots, starts + + +def low_ratio_index_topk_hip_extend( + backend, + layer, + x, + q_lora, + pos, + forward_batch: ForwardBatch, +) -> None: + """Ragged prefill: the FlyDSL prefill kernel scores every token's visible compressed positions, + candidate masks are published or applied per request, one paged top-k selects every row. + Identity requests are written without scores; an all-identity batch skips the kernel.""" + pool = backend.token_to_kv_pool + metadata = backend.forward_metadata + core = metadata.core_metadata + ratio = layer.compress_ratio + indexer = layer.indexer + page_indices = core.sparse_page_indices(ratio) + raw_indices = core.sparse_raw_indices(ratio) + page_indices.fill_(-1) + if raw_indices is not None: + raw_indices.fill_(-1) + + seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu) + # bounded replay counts cover only each request's tail + if metadata.late_layer_tail is not None: + extend_lens_cpu = metadata.late_layer_tail.extend_seq_lens_cpu + else: + extend_lens_cpu = _as_int_list(forward_batch.extend_seq_lens_cpu) + assert seq_lens_cpu is not None and extend_lens_cpu is not None + lc_per_req = [s // ratio for s in seq_lens_cpu] + if pos.numel() == 0 or not any(lc_per_req): + if indexer.is_candidate_source: + backend.candidate_masks = [None] * len(lc_per_req) + return + + if backend.low_ratio_identity_skip: + # identity: every row selects all its visible compressed positions + is_identity = [s // ratio <= indexer.index_topk for s in seq_lens_cpu] + else: + is_identity = [False] * len(seq_lens_cpu) + compress_lens = ((pos + 1) // ratio).to(torch.int32) + consume = backend.candidate_masks if indexer.uses_candidates else None + publish = [] if indexer.is_candidate_source else None + if any(is_identity): + # only the score-free fill resolves slots itself; scored rows resolve in the top-k transform + k_slots, starts = _extend_k_slots( + backend.req_to_token, + ratio=ratio, + lc_per_req=lc_per_req, + req_pool_indices=forward_batch.req_pool_indices.to(torch.int64), + device=pos.device, + ) + + def fill_identity_requests(req_lo, req_hi, tok_lo): + """Identity requests req_lo..req_hi: rows written without scores, None published per request.""" + tok = tok_lo + for b in range(req_lo, req_hi): + t_len, lc = extend_lens_cpu[b], lc_per_req[b] + rows = slice(tok, tok + t_len) + tok += t_len + if lc == 0: + if publish is not None: + publish.append(None) + continue + _fill_identity_request( + lc=lc, + slots_j=k_slots[starts[b] : starts[b] + lc], + lens=compress_lens[rows], + page_rows=page_indices[rows], + raw_rows=None if raw_indices is None else raw_indices[rows], + ) + if publish is not None: + publish.append(None) + + if all(is_identity): + fill_identity_requests(0, len(extend_lens_cpu), 0) + if publish is not None: + backend.candidate_masks = publish + return + + indexer_metadata = metadata.low_ratio_indexer_metadata(ratio) + assert indexer_metadata is not None, f"no prefill indexer metadata for {ratio = }" + num_tokens = pos.shape[0] + assert indexer_metadata.page_table.shape[0] >= num_tokens + q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos) + k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id) + k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id) + prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio) + + def score_rows(rows: slice) -> torch.Tensor: + return aiter_fp4_paged_mqa_logits( + q_fp4=q_fp4[rows], + q_scale=q_scale[rows], + k_payload=k_payload, + k_scale=k_scale, + weights=weights[rows], + page_table=indexer_metadata.page_table[rows], + c4_seq_lens=indexer_metadata.compressed_seq_lens[rows], + weight_scale=1.0, + page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET, + is_decode=False, + prefill_workspace=prefill_workspace if rows.start == 0 else None, + ) + + def select_rows( + rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish + ): + _select_topk_extend_hip( + indexer=indexer, + logits=score_rows(rows), + lc_per_req=lc_per_req[req_lo:req_hi], + extend_lens_cpu=[rows.stop - rows.start] + if req_hi == req_lo + 1 + else extend_lens_cpu[req_lo:req_hi], + is_identity=group_is_identity, + compress_lens=compress_lens[rows], + page_table=indexer_metadata.page_table[rows], + page_size=indexer_metadata.compressed_page_size, + page_indices=page_indices[rows], + raw_indices=raw_indices[rows] if raw_indices is not None else None, + consume=consume_rows, + publish=publish, + ) + + # group requests to fit the pooled logits block; rows are independent, so grouping is exact + rows_per_chunk = logits_rows_per_chunk( + indexer_metadata.page_table, LOW_RATIO_PAGE_TABLE_BUCKET + ) + groups = _request_groups(extend_lens_cpu, rows_per_chunk) + for req_lo, req_hi, tok_lo, tok_hi in groups: + group_is_identity = is_identity[req_lo:req_hi] + if all(group_is_identity): + fill_identity_requests(req_lo, req_hi, tok_lo) + continue + if tok_hi - tok_lo <= rows_per_chunk: + select_rows( + slice(tok_lo, tok_hi), + req_lo, + req_hi, + group_is_identity, + None if consume is None else consume[req_lo:req_hi], + publish, + ) + continue + # a request wider than the logits block: scored in row chunks, its mask published in one piece + assert req_hi == req_lo + 1, (req_lo, req_hi) + pieces = [] + for lo in range(tok_lo, tok_hi, rows_per_chunk): + rows = slice(lo, min(lo + rows_per_chunk, tok_hi)) + piece = [] if publish is not None else None + consume_rows = None + if consume is not None: + consume_rows = [ + None + if consume[req_lo] is None + else slice_candidate_blocks( + consume[req_lo], slice(lo - tok_lo, rows.stop - tok_lo) + ) + ] + select_rows(rows, req_lo, req_hi, group_is_identity, consume_rows, piece) + if piece: + pieces.append(piece[0]) + if publish is not None: + publish.append(cat_candidate_blocks(pieces)) + if publish is not None: + backend.candidate_masks = publish + + +def _select_topk_extend_hip( + *, + indexer, + logits: torch.Tensor, + lc_per_req: List[int], + extend_lens_cpu: List[int], + is_identity: List[bool], + compress_lens: torch.Tensor, + page_table: torch.Tensor, + page_size: int, + page_indices: torch.Tensor, + raw_indices: Optional[torch.Tensor], + consume: Optional[List[Optional[CandidateBlocks]]], + publish: Optional[List[Optional[CandidateBlocks]]], +) -> None: + """Row t of logits scores its request's compressed positions in columns 0..lc-1, reachable + up to compress_lens[t]. A source layer publishes one CandidateBlocks per request (None + for identity / empty), a consumer selects inside its published blocks, others run one paged top-k.""" + assert page_indices.shape[1] == indexer.index_topk, ( + f"the paged top-k selects page_indices.shape[1] = {page_indices.shape[1]} " + f"slots, the indexer wants {indexer.index_topk}" + ) + if publish is not None: + tok_start = 0 + for b, (lc, t_len) in enumerate(zip(lc_per_req, extend_lens_cpu)): + rows = slice(tok_start, tok_start + t_len) + tok_start += t_len + if lc == 0 or t_len == 0 or is_identity[b]: + # Consumers index the publication by request, so keep the slot. + publish.append(None) + continue + step = max(1, _CANDIDATE_SCORE_BUDGET_BYTES // (lc * 4)) + pieces = [] + for lo in range(0, t_len, step): + chunk = slice(rows.start + lo, rows.start + min(lo + step, t_len)) + pieces.append( + select_candidate_blocks_hip( + logits[chunk, :lc], + compress_lens[chunk], + topk_blocks=indexer.candidate_topk_blocks, + block_size=indexer.candidate_block_size, + ) + ) + publish.append(cat_candidate_blocks(pieces)) + if consume is not None: + tok_start = 0 + for b, t_len in enumerate(extend_lens_cpu): + rows = slice(tok_start, tok_start + t_len) + tok_start += t_len + if t_len == 0: + continue + rows_page = page_indices[rows] + rows_raw = raw_indices[rows] if raw_indices is not None else None + if consume[b] is None: + topk_transform_paged_sorted( + logits[rows], + compress_lens[rows].contiguous(), + page_table[rows], + rows_page, + page_size, + rows_raw, + ) + else: + topk_within_candidate_blocks_hip( + logits[rows], + compress_lens[rows], + consume[b], + page_table=page_table[rows], + page_size=page_size, + page_indices=rows_page, + raw_indices=rows_raw, + sort_output=True, + ) + return + topk_transform_paged_sorted( + logits, + compress_lens.contiguous(), + page_table, + page_indices, + page_size, + raw_indices, + ) + + +def _request_groups( + extend_lens_cpu: List[int], rows_per_chunk: int +) -> List[Tuple[int, int, int, int]]: + """Consecutive request groups whose token rows fit rows_per_chunk (a single + request always forms a group): (req_lo, req_hi, tok_lo, tok_hi).""" + groups = [] + req_lo, tok_lo, tok = 0, 0, 0 + for r, t_len in enumerate(extend_lens_cpu): + if r > req_lo and tok + t_len - tok_lo > rows_per_chunk: + groups.append((req_lo, r, tok_lo, tok)) + req_lo, tok_lo = r, tok + tok += t_len + groups.append((req_lo, len(extend_lens_cpu), tok_lo, tok)) + return groups + + +def store_index_k_split( + pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id: int +) -> None: + """The c1 / c2 decode index-K store into ROCm's split pools: payload and scales in + two buffers, where CUDA packs one fused [.., 68]-byte row.""" + index_k_norm_rope_pack_store_split( + indexer.forward_wk(latent), + indexer.k_norm.weight.data, + indexer.k_norm.eps, + freqs_cis, + pos, + out_loc, + pool.get_index_k_fp4_payload_buffer(layer_id), + pool.get_index_k_fp4_scale_buffer(layer_id), + ratio=layer.compress_ratio, + ) diff --git a/python/sglang/srt/layers/attention/graph_variants.py b/python/sglang/srt/layers/attention/graph_variants.py index 6c34620d23..05673607be 100644 --- a/python/sglang/srt/layers/attention/graph_variants.py +++ b/python/sglang/srt/layers/attention/graph_variants.py @@ -98,7 +98,7 @@ def create_dsv41_candidate_graph_variants( import torch from sglang.srt.model_executor.forward_batch_info import ForwardMode - from sglang.srt.utils import is_hip + from sglang.srt.utils import is_gfx95_supported, is_hip text_config = model_runner.model_config.hf_text_config dspark_target_verify = ( @@ -110,8 +110,11 @@ def create_dsv41_candidate_graph_variants( if not ( (capture_forward_mode == ForwardMode.DECODE or dspark_target_verify) and model_runner.device == "cuda" - and not is_hip() - and torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10 + and ( + is_gfx95_supported() + if is_hip() + else torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10 + ) and getattr(text_config, "model_type", None) == "deepseek_v41" and getattr(text_config, "candidate_source_layer_id", -1) >= 0 ): diff --git a/python/sglang/srt/layers/attention/hip_flash_mla.py b/python/sglang/srt/layers/attention/hip_flash_mla.py index 4576873a7d..432b13a9a1 100644 --- a/python/sglang/srt/layers/attention/hip_flash_mla.py +++ b/python/sglang/srt/layers/attention/hip_flash_mla.py @@ -1,17 +1,207 @@ -from typing import Any, Optional +from typing import Any, Optional, Tuple import torch +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.environ import envs -from sglang.srt.utils import is_hip +from sglang.srt.runtime_context import get_exec +from sglang.srt.utils import is_gfx95_supported, is_hip FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn +_HIP_BACKENDS = ("tilelang", "triton", "aiter_sparse", "torch", "comparison") + +# at this many query tokens and above the aiter sparse kernel runs unsplit (batch-invariant prefill) +_AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS = 1024 + + +def hip_attn_kv_splits() -> int: + """Split-KV count for the HIP sparse decode kernels. Deterministic inference pins 4 + splits (a fixed combine order keeps the bits identical at every batch size) and + everything else uses 0: adaptive splits plus the native 16-head attention for small + TP4 decode batches.""" + try: + deterministic = get_exec().deterministic.enable_deterministic_inference + except ValueError: # no published exec config (kernel tests, offline tools) + deterministic = False + return 4 if deterministic else 0 + + +# the eager warmups before each capture fill this, so captured graphs reuse these tensors +_UNIFORM_INDPTR_CACHE: dict[tuple[int, int, str], torch.Tensor] = {} + + +def _uniform_indptr(num_tokens: int, width: int, device: str) -> torch.Tensor: + """Row pointers of the aiter sparse decode kernel: token t reads kv_indices[t*w : (t+1)*w]. + + Only tensors built outside capture are cached: an arange recorded into a graph runs only + when that graph replays, and V4.1 captures two verify graphs per batch size.""" + key = (num_tokens, width, device) + indptr = _UNIFORM_INDPTR_CACHE.get(key) + if indptr is not None: + return indptr + indptr = torch.arange( + 0, (num_tokens + 1) * width, width, dtype=torch.int32, device=device + ) + if ( + num_tokens < _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS + and not torch.cuda.is_current_stream_capturing() + ): + _UNIFORM_INDPTR_CACHE[key] = indptr + return indptr + + +def aiter_sparse_decode_fwd( + q: torch.Tensor, + k_cache: torch.Tensor, + indices: torch.Tensor, + attn_sink: torch.Tensor, + softmax_scale: float, + extra_k_cache: Optional[torch.Tensor] = None, + extra_indices_in_kvcache: Optional[torch.Tensor] = None, + inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + topk_length: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, + **_, +): + """aiter's gfx950 gluon sparse decode kernel (pa_decode_sparse) behind the + flash_mla_with_kvcache shapes. Only -1 entries are skipped, so callers fold + topk_length into the index lists; inv_rope folds the model's inverse RoPE into the output.""" + if k_cache.shape[-1] in ( + KVLayout.V41.bytes_per_token, + KVLayout.V41_FP4.bytes_per_token, + ): + from sglang.kernels.ops.attention.dsv4.compact_attention_hip import ( + compact_attention_hip, + ) + + b, s, h, d = q.shape + out = compact_attention_hip( + q.reshape(b * s, h, d), + k_cache, + indices, + topk_length.reshape(-1), + attn_sink, + extra_cache=extra_k_cache, + extra_indices=extra_indices_in_kvcache, + extra_lengths=( + extra_topk_length.reshape(-1) if extra_k_cache is not None else None + ), + softmax_scale=softmax_scale, + inv_rope=inv_rope, + splits=( + 1 + if b * s >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS + else hip_attn_kv_splits() or None + ), + ) + return out.view(b, s, h, d), None + + from aiter.ops.triton.attention.pa_decode_sparse import pa_decode_sparse + + from sglang.kernels.ops.attention.sparse_decode_reduce_hip import ( + gfx_sparse_split_reduce, + ) + + b, s, h, d = q.shape + n = b * s + q3 = q.reshape(n, h, d) + assert k_cache.dim() == 4 and k_cache.shape[2] == 1, k_cache.shape + cache = k_cache.view(torch.uint8).squeeze(2) + idx = indices.reshape(-1) + assert idx.dtype == torch.int32 and idx.is_contiguous() + indptr = _uniform_indptr(n, indices.shape[-1], str(q.device)) + extra_kwargs = {} + if extra_k_cache is not None: + assert extra_k_cache.dim() == 4 and extra_k_cache.shape[2] == 1 + extra_idx = extra_indices_in_kvcache.reshape(-1) + assert extra_idx.dtype == torch.int32 and extra_idx.is_contiguous() + extra_kwargs = dict( + extra_cache=extra_k_cache.view(torch.uint8).squeeze(2), + extra_indices=extra_idx, + extra_indptr=_uniform_indptr( + n, extra_indices_in_kvcache.shape[-1], str(q.device) + ), + ) + if n >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS: + extra_kwargs["kv_splits"] = 1 + elif hip_attn_kv_splits() > 0: + # a pinned split count keeps the combine order, hence the bits, the same at every batch size + extra_kwargs["kv_splits"] = hip_attn_kv_splits() + out = pa_decode_sparse( + q3, + cache, + idx, + indptr, + attn_sink, + softmax_scale, + skip_reduce=True, + **extra_kwargs, + ) + if isinstance(out, tuple): + # Split-KV partials (acc, m, l): combine them here. + part_acc, part_m, part_l = out + out = gfx_sparse_split_reduce( + part_acc, part_m, part_l, attn_sink, q.dtype, inv_rope=inv_rope + ) + elif inv_rope is not None: + _apply_inverse_rope(out, inv_rope) + return out.view(b, s, h, d), None + + +def _apply_inverse_rope( + out: torch.Tensor, inv_rope: Tuple[torch.Tensor, torch.Tensor] +) -> None: + """The model's standalone inverse RoPE on out [n, h, d] (last 64 dims of + every head), for the paths that did not fold it into their combine.""" + from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace + + freqs_real, positions = inv_rope + n = out.shape[0] + # freqs_real is view_as_real(freqs_cis).flatten(-2); fused_rope_inplace takes the complex table + freqs_cis = torch.view_as_complex(freqs_real.view(freqs_real.shape[0], -1, 2)) + fused_rope_inplace( + out.view(n, -1, out.shape[-1])[..., -freqs_real.shape[-1] :], + None, + freqs_cis, + positions.view(-1)[:n], + inverse=True, + ) + + +def resolve_hip_flashmla_backend(backend: Optional[str] = None) -> str: + """The HIP decode attention kernel name; "auto" (the default) is aiter's + gluon sparse kernel on gfx950 and the tilelang partial + combine elsewhere.""" + if backend is None: + backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get() + if backend == "auto": + return "aiter_sparse" if is_gfx95_supported() else "tilelang" + if backend == "aiter_sparse" and not is_gfx95_supported(): + raise ValueError( + "SGLANG_HACK_FLASHMLA_BACKEND=aiter_sparse needs gfx950 (aiter's gluon sparse " + "decode kernel); use auto or tilelang" + ) + return backend + + +# kernels taking the real per-rank head count; the tilelang kernel builds only for 64-padded widths +_HIP_BACKENDS_ANY_HEAD_COUNT = frozenset({"aiter_sparse", "triton"}) + + +def hip_attention_needs_head_pad() -> bool: + """Whether the kernel DeepseekV4HipRadixBackend.forward picks needs the per-rank query + heads padded to 64 (zero q, zero sink).""" + return resolve_hip_flashmla_backend() not in _HIP_BACKENDS_ANY_HEAD_COUNT + + def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): if is_hip(): - backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get() + # a CUDA backend name (the "comparison" recursion passes "kernel") takes the HIP default + backend = resolve_hip_flashmla_backend( + backend if backend in _HIP_BACKENDS else None + ) else: import sgl_kernel.flash_mla as flash_mla @@ -44,6 +234,9 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): return triton_fp8_attention_fwd(**kwargs) + if backend == "aiter_sparse": + return aiter_sparse_decode_fwd(**kwargs) + if backend == "kernel": return flash_mla.flash_mla_with_kvcache(**kwargs) diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 0cb2e953c1..3e14388d59 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -33,6 +33,7 @@ from sglang.kernels.ops.embeddings.engram_hash import ( engram_hash_ids_and_commit, ) from sglang.srt.distributed import tensor_model_parallel_all_reduce +from sglang.srt.distributed.parallel_state import inplace_all_reduce from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import ( attn_cp_all_gather_into_tensor, @@ -47,7 +48,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_model, get_parallel, get_serving -from sglang.srt.utils import add_prefix, is_cuda +from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer logger = logging.getLogger(__name__) @@ -55,10 +56,13 @@ logger = logging.getLogger(__name__) _MILLER_RABIN_WITNESSES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37) +_is_hip = is_hip() + def _cuda_kernels(t: torch.Tensor) -> bool: - """True where the Triton kernels apply; ROCm and CPU take the torch paths.""" - return t.is_cuda and is_cuda() + """True where the Triton kernels apply (CUDA and gfx950); other ROCm GPUs and CPU + take the torch paths.""" + return t.is_cuda and (is_cuda() or is_gfx95_supported()) def _is_prime(n: int) -> bool: @@ -775,9 +779,18 @@ class EngramEmbedding(nn.Module): return self._empty(indices) values = self._owned_rows(indices) if self.tp_size > 1: - values = tensor_model_parallel_all_reduce(values) + values = self._reduce_owned_rows(values) return values + def _reduce_owned_rows(self, values: torch.Tensor) -> torch.Tensor: + if _is_hip and values.is_cuda: + # Integer addition preserves all BF16 bits because exactly one shard owns each row. + inplace_all_reduce( + values.view(torch.int32), group_name=get_parallel().tp_group.unique_name + ) + return values + return tensor_model_parallel_all_reduce(values) + def _empty(self, indices: torch.Tensor) -> torch.Tensor: return torch.empty( *indices.shape, self.dim, dtype=torch.bfloat16, device=indices.device @@ -833,9 +846,14 @@ class EngramEmbedding(nn.Module): and self.tp_size == get_parallel().attn_dp_size and rows == self.tp_size * local.shape[0] ) or is_dp_gatherv_active(): - dp_reduce_scatter_tensor(local, values) + if _is_hip and values.is_cuda: + dp_reduce_scatter_tensor( + local.view(torch.int32), values.view(torch.int32) + ) + else: + dp_reduce_scatter_tensor(local, values) else: - dp_scatter(local, tensor_model_parallel_all_reduce(values), forward_batch) + dp_scatter(local, self._reduce_owned_rows(values), forward_batch) return local.view(*indices.shape, self.dim) diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index c827500704..25c6a34a07 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -233,6 +233,8 @@ if _use_aiter: from aiter.fused_moe import fused_topk as aiter_fused_topk except ImportError: raise ImportError("aiter is required when SGLANG_USE_AITER is set to True") + from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_reduce_partials + from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_gate if _is_musa: try: from mate import moe_fused_gate @@ -641,7 +643,10 @@ class TopK(BaseFusedOp): num_token_non_padded: Optional[torch.Tensor] = None, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, + router_logits_partials: Optional[torch.Tensor] = None, ) -> TopKOutput: + if router_logits_partials is not None: + rocm_router_reduce_partials(router_logits_partials, router_logits) self.topk_config.torch_native = True topk_output = select_experts( hidden_states=hidden_states, @@ -662,7 +667,11 @@ class TopK(BaseFusedOp): num_token_non_padded: Optional[torch.Tensor] = None, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, + router_logits_partials: Optional[torch.Tensor] = None, ) -> TopKOutput: + """router_logits_partials (ROCm decode router): fp32 split-K partials whose + fixed-order sum is the logits; router_logits is then a buffer the fused gate + fills, and any other reader first reduces the partials into it.""" if dynamic_expert_bias is not None: output_format = TopKOutputFormat.STANDARD elif self.topk_config.output_format is not None: @@ -688,6 +697,13 @@ class TopK(BaseFusedOp): else: output_format = TopKOutputFormat.STANDARD + if ( + router_logits_partials is not None + and output_format != TopKOutputFormat.STANDARD + ): + rocm_router_reduce_partials(router_logits_partials, router_logits) + router_logits_partials = None + if output_format == TopKOutputFormat.TRITON_KERNEL: # renormalize=True is equivalent to sm_first=False ( @@ -729,6 +745,7 @@ class TopK(BaseFusedOp): num_token_non_padded=num_token_non_padded, expert_location_dispatch_info=expert_location_dispatch_info, dynamic_expert_bias=dynamic_expert_bias, + router_logits_partials=router_logits_partials, ) return self._apply_waterfill(topk_output, hidden_states.shape[0]) @@ -1423,11 +1440,23 @@ def biased_topk_jit_kernel_impl( apply_routed_scaling_factor_on_output: Optional[bool] = False, packed_out: Optional[torch.Tensor] = None, sqrtsoftplus_log1p: bool = False, + router_logits_partials: Optional[torch.Tensor] = None, ) -> Tuple[torch.Tensor, torch.Tensor]: assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0: assert packed_out is None, "aiter topk_gating cannot emit packed ids" + if router_logits_partials is not None: + # ROCm decode router: split-K reduce + gate in one launch + return rocm_router_gate( + gating_output, + correction_bias, + topk, + renormalize, + routed_scaling_factor, + partials=router_logits_partials, + ) + from aiter import topk_gating num_tokens = gating_output.shape[0] @@ -1451,6 +1480,7 @@ def biased_topk_jit_kernel_impl( return topk_weights, topk_ids else: + assert router_logits_partials is None from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate # DeepSeek-V4 stores e_score_correction_bias in bf16 (for the aiter @@ -2460,6 +2490,7 @@ def select_experts( num_token_non_padded: Optional[torch.Tensor] = None, expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, dynamic_expert_bias: Optional[torch.Tensor] = None, + router_logits_partials: Optional[torch.Tensor] = None, ) -> StandardTopKOutput: top_k = topk_config.top_k use_grouped_topk = topk_config.use_grouped_topk @@ -2495,6 +2526,22 @@ def select_experts( ) routing_overridden = simulate_uniform_experts or simulate_round_robin_experts + if router_logits_partials is not None and not ( + _use_aiter + and scoring_func == "sqrtsoftplus" + and custom_routing_function is None + and not use_grouped_topk + and not torch_native + and expert_location_dispatch_info is None + and ( + num_fused_shared_experts == 0 + or has_per_rank_fused_shared_slots(num_fused_shared_experts) + ) + ): + # only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits + rocm_router_reduce_partials(router_logits_partials, router_logits) + router_logits_partials = None + ( router_logits, correction_bias, @@ -2619,6 +2666,8 @@ def select_experts( _packed_kwargs = dict(packed_out=packed_topk) if topk_config.sqrtsoftplus_log1p: _packed_kwargs["sqrtsoftplus_log1p"] = True + if router_logits_partials is not None: + _packed_kwargs["router_logits_partials"] = router_logits_partials topk_weights, topk_ids = _biased_topk( hidden_states=hidden_states, gating_output=router_logits, diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 25b39ae95e..026046c729 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -126,6 +126,9 @@ _is_cpu_amx_available = cpu_has_amx_support() _is_cpu = is_cpu() _is_fp8_fnuz = is_fp8_fnuz() _is_gfx95_supported = is_gfx95_supported() + +if _is_hip: + from sglang.srt.layers.quantization import fp8_hip # gfx942 (MI300) has no MX matmul HW; MXFP8 checkpoints are converted to # block-fp8 [128,128] at load and run through the native block-fp8 kernels. # SGLANG_FORCE_MXFP8_BLOCK_CONVERT=1 opts into that same block-fp8 path on @@ -951,6 +954,8 @@ class Fp8LinearMethod(LinearMethodBase): "weight_scale_inv_swizzled", block_scale_interleave(scale_u8.contiguous()).contiguous(), ) + elif backend.is_gfx95(): + fp8_hip.process_dense_weights(self, layer, scale_u8) elif backend.is_deep_gemm(): from sglang.srt.layers.deep_gemm_wrapper.configurer import ( DEEPGEMM_SCALE_UE8M0, @@ -1162,6 +1167,9 @@ class Fp8LinearMethod(LinearMethodBase): bias=bias, ) + if _is_hip and self.block_fp8_as_mxfp8 and self.mxfp8_dense_backend.is_gfx95(): + return fp8_hip.apply_dense(self, layer, x, bias) + mxfp8_view = self.use_mxfp8 or ( self.block_fp8_as_mxfp8 and layer.block_fp8_mxfp8_ready ) diff --git a/python/sglang/srt/layers/quantization/fp8_hip.py b/python/sglang/srt/layers/quantization/fp8_hip.py new file mode 100644 index 0000000000..9e5ea1e2d0 --- /dev/null +++ b/python/sglang/srt/layers/quantization/fp8_hip.py @@ -0,0 +1,126 @@ +"""gfx950 dense routes of Fp8LinearMethod for 32x32-block fp8 checkpoints served as +MXFP8 (block_fp8_as_mxfp8): the Triton dot_scaled kernel, or the native scaled-MFMA +kernels on a lane-ordered weight. The fused gfx950 producers hand these routes their +operand as an Fp8GridActivation (bf16 already on the fp8 grid) or an Mxfp8Activation.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + Fp8GridActivation, + Mxfp8Activation, + bf16_dequant_blockscaled_linear, + dequant_block_fp8_weight_to_bf16, + dequant_mxfp8_to_bf16, +) +from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import ( + native_route_supports, + prepare_mxfp8_native_weight, +) +from sglang.srt.layers.utils import copy_or_rebind_param + + +def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None: + """The gfx950 branch of Fp8LinearMethod.process_weights_after_loading.""" + backend = method.mxfp8_dense_backend + if backend.is_gfx95_dot_scaled(): + # dot_scaled reads canonical [N, K // 32] e8m0 bytes; block scales stay for direct readers + if scale_u8 is not None: + copy_or_rebind_param(layer, "weight_scale_inv_mx", scale_u8.contiguous()) + return + assert backend.is_gfx95_mxfp8_native() + n, k = layer.weight.shape + layer.mxfp8_native_ready = False + if native_route_supports(n, k): + # same bytes in scaled-MFMA lane order + shuffled, scale_ue8m0 = prepare_mxfp8_native_weight( + layer.weight.data, + layer.weight_scale_inv.data, + method.weight_block_size, + ) + copy_or_rebind_param(layer, "weight", shuffled.view(torch.float8_e4m3fn)) + copy_or_rebind_param(layer, "weight_scale_mx_e8m0", scale_ue8m0) + layer.mxfp8_native_ready = True + else: + # a shape the native kernels do not tile keeps the bf16-dequant route + copy_or_rebind_param( + layer, + "weight_bf16", + dequant_block_fp8_weight_to_bf16( + layer.weight.data, + layer.weight_scale_inv.data, + method.weight_block_size, + ), + ) + + +def apply_dense( + method, layer: torch.nn.Module, x, bias: Optional[torch.Tensor] +) -> torch.Tensor: + """Fp8LinearMethod.apply on a gfx950 route. x is a bf16 tensor, an (fp8, scale) + tuple from a fused quant kernel, or one of the wrappers the fused producers emit.""" + backend = method.mxfp8_dense_backend + mxfp8_ready = layer.block_fp8_mxfp8_ready + native_route = mxfp8_ready and backend.is_gfx95_mxfp8_native() + # Unwrap the producer's operand: the native route takes fp8 + scales or the fp8-grid + # bf16 directly; the dot_scaled route quantizes the plain tensor itself (per-32 + # rounding is idempotent, so the wrapper's rounding is exact for it). + input_scale, on_fp8_grid = None, False + if isinstance(x, Mxfp8Activation): + if native_route: + x, input_scale = x.q, x.scale + else: + x = dequant_mxfp8_to_bf16(x.q, x.scale) + elif isinstance(x, Fp8GridActivation): + x, on_fp8_grid = x.x, native_route + elif isinstance(x, tuple): + x, input_scale = x + if native_route: + return _apply_native(method, layer, x, bias, input_scale, on_fp8_grid) + if mxfp8_ready and input_scale is None: + return method.w8a8_mxfp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale_inv_mx, + input_scale=None, + bias=bias, + ) + # a shape the gfx950 kernels do not tile, or a pre-quantized tuple off the native route + return method.w8a8_block_fp8_linear( + input=x, + weight=layer.weight, + block_size=method.weight_block_size, + weight_scale=layer.weight_scale_inv, + input_scale=input_scale, + bias=bias, + ) + + +def _apply_native( + method, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor], + input_scale: Optional[torch.Tensor] = None, + input_on_fp8_grid: bool = False, +) -> torch.Tensor: + """The native route (mxfp8_native_amd_gfx95); a layer whose shape the native + kernels do not tile keeps the bf16-dequant route.""" + if layer.mxfp8_native_ready: + # an fp8-grid bf16 input re-encodes exactly, so it needs no flag here + return method.w8a8_mxfp8_linear( + input=x, + weight_shuffled=layer.weight.view(torch.uint8), + weight_scale_ue8m0=layer.weight_scale_mx_e8m0, + input_scale=input_scale, + bias=bias, + ) + if input_scale is not None: + x = dequant_mxfp8_to_bf16(x, input_scale) + input_on_fp8_grid = True + return bf16_dequant_blockscaled_linear( + x, layer.weight_bf16, bias=bias, input_on_fp8_grid=input_on_fp8_grid + ) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 7331504a64..04b542a549 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -381,6 +381,8 @@ class Mxfp8DenseGemmBackend(Enum): FLASHINFER_TRTLLM = "flashinfer_trtllm" DEEP_GEMM = "deep_gemm" GFX95_DOT_SCALED = "gfx95_dot_scaled" + # gfx950 native MXFP8: lane-ordered fp8 weight + ue8m0 scale bytes; untiled shapes keep bf16 + GFX95_MXFP8_NATIVE = "gfx95_mxfp8_native" UNSUPPORTED = "unsupported" def is_flashinfer_cutlass(self) -> bool: @@ -401,6 +403,15 @@ class Mxfp8DenseGemmBackend(Enum): def is_gfx95_dot_scaled(self) -> bool: return self == Mxfp8DenseGemmBackend.GFX95_DOT_SCALED + def is_gfx95_mxfp8_native(self) -> bool: + return self == Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE + + def is_gfx95(self) -> bool: + return self in ( + Mxfp8DenseGemmBackend.GFX95_DOT_SCALED, + Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE, + ) + def is_unsupported(self) -> bool: return self == Mxfp8DenseGemmBackend.UNSUPPORTED @@ -724,8 +735,14 @@ def _unsupported_mxfp8_linear(*args, **kwargs) -> torch.Tensor: def resolve_block_fp8_mxfp8_backend() -> Mxfp8DenseGemmBackend: - """The FlashInfer MXFP8 backend a 32-wide-K ue8m0 block-fp8 weight can run on.""" + """The MXFP8 backend (FlashInfer on CUDA, gfx950 on ROCm) a 32-wide-K ue8m0 + block-fp8 weight can run on.""" backend = get_fp8_gemm_runner_backend() + # the Triton block kernel's ue8m0 activation quant is CUDA-only, so gfx950 takes an MXFP8 route + if _is_hip and _is_gfx95_supported: + if backend.is_triton(): + return Mxfp8DenseGemmBackend.GFX95_DOT_SCALED + return Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE # Explicit CUTLASS / CuTe-DSL only: they leave the weight untouched and store # the swizzled scale separately, so the block layout stays readable by Triton. if not (backend.is_flashinfer_cutedsl() or backend.is_flashinfer_cutlass()): @@ -758,6 +775,18 @@ def dispatch_block_fp8_mxfp8_linear(backend: Mxfp8DenseGemmBackend) -> Callable: return partial( flashinfer_mxfp8_blockscaled_linear, backend="cute-dsl", pin_tactic=True ) + if backend.is_gfx95_dot_scaled(): + from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + dot_scaled_mxfp8_blockscaled_linear, + ) + + return dot_scaled_mxfp8_blockscaled_linear + if backend.is_gfx95_mxfp8_native(): + from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import ( + mxfp8_native_blockscaled_linear, + ) + + return mxfp8_native_blockscaled_linear return _unsupported_mxfp8_linear diff --git a/python/sglang/srt/layers/rocm_linear_utils.py b/python/sglang/srt/layers/rocm_linear_utils.py index ba56fd6cd1..b2042e5161 100644 --- a/python/sglang/srt/layers/rocm_linear_utils.py +++ b/python/sglang/srt/layers/rocm_linear_utils.py @@ -1,8 +1,12 @@ +from typing import Optional, Tuple + import torch from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat from aiter.tuned_gemm import tgemm +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.srt.runtime_context import get_exec from sglang.srt.utils import is_gfx95_supported __all__ = [ @@ -31,6 +35,27 @@ def aiter_dsv3_router_gemm( return tgemm.mm(hidden_states, weight.detach(), otype=hidden_states.dtype) +def rocm_dsv3_router_split_k( + gate, hidden_states: torch.Tensor +) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """The ROCm decode router for gate (a MoEGate): an fp32 logits buffer plus + the split-K partials whose fixed-order sum fills it, or None when gate.forward + applies. Only TopK.forward_cuda(..., router_logits_partials=partials) may read + the buffer: it sums the partials into it inside the fused gate launch.""" + num_tokens = hidden_states.shape[0] + if not 0 < num_tokens <= gate.rocm_router_max_tokens: + return None + if get_exec().deterministic.enable_deterministic_inference: + return None + partials = rocm_router_gemv_split_k(hidden_states, gate.weight) + logits = torch.empty( + (num_tokens, gate.weight.shape[0]), + dtype=torch.float32, + device=hidden_states.device, + ) + return logits, partials + + def get_dsv3_gemm_output_zero_allocator_size( n_routed_experts: int, num_moe_layers: int, allocate_size: int, embedding_dim: int ): diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index ba5818284e..859de5a5bf 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -23,11 +23,12 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import ( from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs +from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backend from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool from sglang.srt.mem_cache.memory_pool import KVCache from sglang.srt.runtime_context import get_exec, get_platform, get_spec -from sglang.srt.utils import ceil_div, is_hip +from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip logger = logging.getLogger(__name__) @@ -170,7 +171,8 @@ def flashmla_supports_v41_kv_layouts() -> bool: def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: """The (main-cache layout, compressed-cache option) for a new DeepSeek-V4 - family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA.""" + family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950 + AITER attention.""" mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower() option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower() if mode == "v4": @@ -181,6 +183,17 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: and torch.version.cuda is not None and torch.cuda.get_device_capability()[0] == 10 ) + if _is_hip: + supported = ( + is_gfx95_supported() and resolve_hip_flashmla_backend() == "aiter_sparse" + ) + if mode == "auto" and not supported: + return KVLayout.V4, None + if not supported: + raise ValueError( + "V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention" + ) + return KVLayout.V41, option supported = flashmla_supports_v41_kv_layouts() if mode == "auto": if is_sm100 and supported: @@ -665,6 +678,21 @@ class DeepSeekV4IndexerPool(KVCache): loc: torch.Tensor, cache_k: torch.Tensor, ) -> None: + if self.uses_aiter_fp4_layout: + # the FlyDSL kernels read the fp4 payload and the packed ue8m0 scales + # from two buffers + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + store_fp4_index_k_cache_split, + ) + + return store_fp4_index_k_cache_split( + cache_k, + self.index_k_payload_buffer[layer_id - self.start_layer], + self.index_k_scale_buffer[layer_id - self.start_layer], + loc, + page_size=self.page_size, + rne=self.index_k_rne, + ) from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( store_fp4_index_k_cache, ) @@ -1894,6 +1922,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): compress_layer_id ) + def low_ratio_index_k_is_split(self, layer_id: int) -> bool: + """Whether the layer's index-K pool keeps payload and scale in the split + FlyDSL layout (ROCm) rather than one fused [.., 68]-byte row.""" + compress_ratio, _, _ = self.layer_mapping[layer_id] + return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout + def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor: self.wait_layer_transfer(layer_id) compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id] @@ -1990,8 +2024,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): eps: float, freqs_cis: torch.Tensor, positions: torch.Tensor, + q: Optional[torch.Tensor] = None, ) -> None: + """q ([B, H, head_dim]): rope its query heads in the same launch.""" if self.uniform_fp8: + assert q is None, "uniform FP8 store does not fuse query RoPE" # Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again), # then an e4m3 cast + scatter with per-tensor scale 1.0. from sglang.kernels.ops.attention.deepseek_v4_rope import ( @@ -2019,6 +2056,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): kvcache=self.get_swa_raw_buffer(layer_id), page_size=self.swa_page_size, layout=self.kv_layout, + q=q, ) def set_unified_key_buffer_radix_fused_norm_rope( diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py new file mode 100644 index 0000000000..ab48d486cb --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_act.py @@ -0,0 +1,53 @@ +"""ROCm activation route of DeepseekV2MLP: aiter's fused clamp + silu-and-mul for 128-wide half +widths (fp8 out for a 128x128-block down_proj), else the Triton silu-and-mul-clamp, whose gfx950 +epilogue lands on down_proj's fp8 grid or emits native MXFP8. Bound by deepseek_v2 under _is_hip.""" + +from __future__ import annotations + +import torch + +from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import ( + silu_and_mul_clamp_fp8_grid_supported, + silu_and_mul_clamp_triton, +) +from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod + + +def resolve_fused_clamp_route(mlp, half_width: int) -> None: + """Fix mlp's activation route from down_proj's loaded weight (once per layer).""" + quant_method = mlp.down_proj.quant_method + # the aiter kernel tiles and quantizes the half width per 128, the 128x128 block GEMM's layout + mlp.use_fused_clamp_act_mul = half_width % 128 == 0 + mlp._fused_clamp_use_fp8 = ( + isinstance(quant_method, Fp8LinearMethod) + and quant_method.block_quant + and quant_method.weight_block_size == [128, 128] + ) + # gfx950 32-block route: the activation lands on the fp8 grid, so down_proj skips its fake-quant + mlp._hip_act_fp8_grid = bool( + isinstance(quant_method, Fp8LinearMethod) + and quant_method.block_fp8_as_mxfp8 + and mlp.down_proj.block_fp8_mxfp8_ready + and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native() + and silu_and_mul_clamp_fp8_grid_supported(half_width) + ) + # the native MXFP8 route takes fp8 + ue8m0 straight from the epilogue at decode token counts + mlp._hip_act_native_consumer = bool( + mlp._hip_act_fp8_grid + and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native() + and mlp.down_proj.mxfp8_native_ready + ) + mlp._fused_clamp_fp8_checked = True + + +def silu_and_mul_clamp(mlp, gate_up: torch.Tensor): + """silu(clamp(g)) * clamp(u) for any half width (unlike the aiter kernel's + multiple of 128), emitted on down_proj's fp8 grid or as native fp8 + ue8m0 + when the resolved route takes it.""" + return silu_and_mul_clamp_triton( + gate_up, + float(mlp.swiglu_limit), + fp8_grid=mlp._hip_act_fp8_grid, + # the native down_proj runs on fp8 + ue8m0 at every M, so hand it those directly + emit_fp8=mlp._hip_act_native_consumer, + ) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py new file mode 100644 index 0000000000..09846c3c6f --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v2_hip_moe.py @@ -0,0 +1,84 @@ +"""ROCm glue of DeepseekV2MoE: V4.1's split-K decode router and the fused all-reduce + mHC +post of the DeepSeek-V4 decode batches.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + +from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post +from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion +from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + ALL_REDUCE_MHC_MAX_ROWS, +) +from sglang.srt.models.deepseek_common.utils import _use_aiter +from sglang.srt.runtime_context import get_parallel + +if _use_aiter: + from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens + from sglang.srt.layers.rocm_linear_utils import rocm_dsv3_router_split_k + + +def router_max_tokens(gate, config, is_hash_moe: bool) -> int: + """Rows up to which the split-K router serves gate (a MoEGate), -1 when it never does: + V4.1 only, DSv4 keeps aiter's router GEMM and gate.""" + if not ( + _use_aiter + and gate.is_deepseek_v4 + and getattr(config, "model_type", None) == "deepseek_v41" + and not is_hash_moe + ): + return -1 + return rocm_router_max_tokens( + num_experts=config.n_routed_experts, + hidden_size=config.hidden_size, + topk=config.num_experts_per_tok, + weight_dtype=gate.weight.dtype, + ) + + +def batch_has_images(forward_batch) -> bool: + """Whether a batch can carry image tokens: only extend batches with image inputs do, so + every other batch takes the fused top-k instead of the vision one.""" + if forward_batch is None: + return True + return ( + forward_batch.forward_mode.is_extend() and forward_batch.contains_image_inputs() + ) + + +def forward_gate( + moe, hidden_states: torch.Tensor, gemm_output_zero_allocator, *, fused_gate: bool +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """The router logits of moe (a DeepseekV2MoE) and, on the split-K decode router, the + partials moe.topk sums into them; fused_gate=False when anything else reads them.""" + if fused_gate and _use_aiter and not moe.is_hash: + logits_and_partials = rocm_dsv3_router_split_k(moe.gate, hidden_states) + if logits_and_partials is not None: + return logits_and_partials + return moe.gate(hidden_states, gemm_output_zero_allocator), None + + +def fused_all_reduce_mhc( + moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor +) -> bool: + """Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the + mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output + and hidden_states only keeps the DP wrapper's tensor contract.""" + if not ( + mhc is not None + and not moe._shared_expert_tp1 + and not mhc.overlap_only + and mhc.post is not None + and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS + ): + return False + mhc.output = all_reduce_mhc_post( + hidden_states, + mhc.residual, + mhc.post, + mhc.comb, + get_parallel().tp_group.ca_comm, + ) + return True diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py index 08f63c1ec9..91b3c1115d 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_fused_mhc.py @@ -1,11 +1,22 @@ import logging -from typing import Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple import torch import triton +if TYPE_CHECKING: + from sglang.kernels.ops.layernorm.mhc_boundary_hip import HcCoefficients + +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_platform +from sglang.srt.layers.moe import get_moe_a2a_backend +from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion +from sglang.srt.runtime_context import ( + get_exec, + get_forward, + get_parallel, + get_platform, +) from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip from sglang.srt.utils.common import is_gfx1250_supported @@ -15,6 +26,22 @@ _is_hip = is_hip() _is_gfx95_supported = is_gfx95_supported() _is_gfx1250_supported = is_gfx1250_supported() +if _is_hip: + from sglang.kernels.ops.communication.all_reduce_mhc_hip import ( + all_reduce_mhc_post, + ) + from sglang.kernels.ops.layernorm.mhc_boundary_hip import ( + hc_boundary_fused_deferred, + ) +if _is_gfx95_supported: + from sglang.srt.models.deepseek_common.amd.deepseek_v4_gfx95_dense import ( + post_attention_norm, + ) + +# the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows +MHC_HIDDEN_SIZE = 5120 +ALL_REDUCE_MHC_MAX_ROWS = 8 + _FUSED_HC_POST_PRE_M_THRESHOLD = 64 _FUSED_HC_POST_PRE_CACHE: dict[tuple, dict[str, torch.Tensor]] = {} _TRITON_MHC_POST_PRE_OPS = None @@ -421,3 +448,178 @@ def apply_mhc_post_pre_boundary( ) post_out = post_out.squeeze(-1) if post_out.ndim == 3 else post_out return residual, layer_input_out, post_out, comb_out, True + + +def hc_boundary( + layer, + x: Optional[torch.Tensor], + residual: torch.Tensor, + post: Optional[torch.Tensor], + comb: Optional[torch.Tensor], + pre_prev: Optional[torch.Tensor], + hc_fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, "HcCoefficients"]: + """Fused sublayer boundary (ROCm): apply the pending hc_post of x onto residual, collapse + with pre_prev (copy 0 when None) and take the mixing statistics. Returns (new_residual, y, + coefficients); the coefficients' reduce + sinkhorn is still pending (see HcCoefficients).""" + new_residual, y, coefficients = hc_boundary_fused_deferred( + x, + residual, + post, + comb, + pre_prev, + hc_fn, + hc_scale, + hc_base, + layer.hc_mult, + layer.hc_sinkhorn_iters, + layer.rms_norm_eps, + layer.hc_eps, + ) + if new_residual is None: + new_residual = residual + if y is None: + y = new_residual[:, 0, :].contiguous() + return new_residual, y, coefficients + + +def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool: + return ( + _is_gfx95_supported + and 1 <= residual.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS + and residual.shape[1:] == (4, MHC_HIDDEN_SIZE) + and residual.dtype == torch.bfloat16 + and residual.is_contiguous() + and layer.config.model_type == "deepseek_v41" + and ( + forward_batch.forward_mode.is_decode() + or forward_batch.forward_mode.is_target_verify() + ) + and get_parallel().attn_dp_size == 1 + and get_parallel().tp_size == 4 + and not layer.dsa_enable_prefill_cp + and not get_forward().sp_active + and not is_batch_invariant_mode_enabled() + and not get_exec().deterministic.enable_deterministic_inference + ) + + +def _make_mhc_fusion( + residual: torch.Tensor, coefficients: "HcCoefficients", comm +) -> Optional[MhcPostFusion]: + # comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off + if comm is None or comm.disabled or not comm.enable_register_for_capturing: + return None + coefficients.materialize() + return MhcPostFusion( + residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre + ) + + +def attention_mhc_fusion(layer, residual, coefficients, forward_batch): + if not ( + _can_fuse_mhc(layer, residual, forward_batch) + and layer.self_attn.attn_tp_size == 4 + and layer.self_attn.wo_b.reduce_results + ): + return None + return _make_mhc_fusion( + residual, coefficients, get_parallel().attn_tp_group.ca_comm + ) + + +def moe_mhc_fusion(layer, residual, coefficients, forward_batch): + if not ( + _can_fuse_mhc(layer, residual, forward_batch) + and layer.mlp.tp_size == 4 + and not layer.mlp._shared_expert_tp1 + and get_moe_a2a_backend().is_none() + ): + return None + return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm) + + +def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None: + # a lazily recorded state still needs its stats before the fused kernel reads them + state.materialize_stats() + if state.stats_stream is not None: + torch.cuda.current_stream().wait_stream(state.stats_stream) + state.output = all_reduce_mhc_post( + x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm + ) + + +def forward_hc_pre_from_prev_fused_boundary( + layer, + positions: torch.Tensor, + hidden_states: Optional[torch.Tensor], + input_ids: torch.Tensor, + forward_batch, + input_ids_global: torch.Tensor, + prev_pre: Optional[torch.Tensor], + pending_post: Optional[Tuple[torch.Tensor, ...]], + defer_post: bool, +) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]: + """ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev. + pending_post is the previous layer's unapplied FFN hc_post (x, residual, post, + comb); with defer_post this layer's is returned the same way and hidden_states is None.""" + if pending_post is None: + pending_post = (None, hidden_states, None, None) + residual, x, attn_coefficients = hc_boundary( + layer, + *pending_post, + prev_pre, + layer.hc_attn_fn, + layer.hc_attn_scale, + layer.hc_attn_base, + ) + # the boundary's reduce + sinkhorn rides in the norm launch + x, x_quant = layer._input_norm( + x, allow_aiter_quant=False, coefficients=attn_coefficients + ) + with layer.self_attn.maybe_use_decode_attn_tp(forward_batch): + mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch) + with use_mhc_post_fusion(mhc): + x = layer.self_attn( + x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant + ) + if mhc is not None: + residual = mhc.output + x = None + residual, x, ffn_coefficients = hc_boundary( + layer, + x, + residual, + attn_coefficients.post if mhc is None else None, + attn_coefficients.comb if mhc is None else None, + attn_coefficients.pre, + layer.hc_ffn_fn, + layer.hc_ffn_scale, + layer.hc_ffn_base, + ) + x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients) + mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch) + with use_mhc_post_fusion(mhc): + x = layer._run_moe_ffn_dp_sync( + x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global + ) + ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors() + if mhc is not None and mhc.output is not None: + # Reduction already applied post. The next boundary consumes this + # materialized residual, including when it would normally defer post. + return mhc.output, ffn_pre, None + if defer_post: + return None, ffn_pre, (x, residual, ffn_post, ffn_comb) + return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None + + +def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients): + """layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where + the gfx950 norm kernel is available; the module norm plus a standalone reduce + + sinkhorn elsewhere.""" + if _is_gfx95_supported: + return post_attention_norm(layer, x, coefficients) + coefficients.materialize() + return layer.post_attention_layernorm(x) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py new file mode 100644 index 0000000000..898bb94509 --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py @@ -0,0 +1,205 @@ +"""gfx950 dense route of the DeepSeek-V4 attention for 32-wide-block (V4.1) fp8 checkpoints: the +fused RMSNorm + fake-quant producers hand wqkv_a / wq_b their operand on the fp8 grid +(Fp8GridActivation) or as native MXFP8 (Mxfp8Activation); deepseek_v4 binds it under _is_hip.""" + +from __future__ import annotations + +import logging +from typing import Optional, Tuple + +import torch +from torch import nn + +from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + Fp8GridActivation, + Mxfp8Activation, +) +from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import ( + rmsnorm_fake_quant_fp8, +) +from sglang.srt.environ import envs +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod +from sglang.srt.layers.quantization.fp8_utils import resolve_block_fp8_mxfp8_backend +from sglang.srt.runtime_context import get_exec +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.srt.utils.common import is_gfx1250_supported + +logger = logging.getLogger(__name__) + +_is_hip = is_hip() +_is_gfx95_supported = is_gfx95_supported() +_is_gfx1250_supported = is_gfx1250_supported() +_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip + +# V4.1's wo_a route on gfx950: an aiter batched GEMM fork with wo_b's fp8-grid rounding in its +# epilogue; None keeps the aiter kernel +_wo_a_fp8_grid_gemm = None +if _use_aiter and _is_gfx95_supported: + from sglang.kernels.ops.gemm.gfx95_batched_gemm_bf16_fp8_grid import ( + batched_gemm_bf16_fp8_grid as _wo_a_fp8_grid_gemm, + ) + + +def fused_rmsnorm_fp8_quant_eligible( + quant_config: Optional[QuantizationConfig], +) -> bool: + """Whether the aiter fused RMSNorm + fp8 quant applies: its (fp8, 128-group scale) + output is consumed only by the 128x128-block dense GEMM.""" + return bool( + _use_aiter + and (_is_gfx95_supported or _is_gfx1250_supported) + and isinstance(quant_config, Fp8Config) + and quant_config.weight_block_size == [128, 128] + ) + + +def fused_rmsnorm_fake_quant_eligible( + quant_config: Optional[QuantizationConfig], +) -> bool: + """Whether rmsnorm_fake_quant_fp8 applies: gfx950 with a 32-wide-block checkpoint + (V4.1), whose dense route takes the norm output already on the fp8 grid as an + Fp8GridActivation.""" + if not (_is_hip and _is_gfx95_supported and isinstance(quant_config, Fp8Config)): + return False + block = quant_config.weight_block_size + return ( + block is not None + and block[1] == 32 + and quant_config.scale_fmt == "ue8m0" + and resolve_block_fp8_mxfp8_backend().is_gfx95_mxfp8_native() + ) + + +def _native_mxfp8_consumer(linear: Optional[nn.Module]) -> bool: + """Whether linear runs the gfx950 native MXFP8 route, which consumes fp8 + ue8m0 + scales directly at every M.""" + if linear is None: + return False + quant_method = linear.quant_method + return ( + isinstance(quant_method, Fp8LinearMethod) + and quant_method.block_fp8_as_mxfp8 + and linear.block_fp8_mxfp8_ready + and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native() + and linear.mxfp8_native_ready + ) + + +def _fake_quant_applies(norm: nn.Module, x: torch.Tensor) -> bool: + return x.dim() == 2 and x.dtype == norm.weight.dtype + + +def q_norm_fake_quant(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]: + """attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b + consumes, already on the fp8 grid); the plain norm, twice, when the rows are not + a 2-D bf16 batch.""" + if not _fake_quant_applies(attn.q_norm, q_lora): + q_lora = attn.q_norm(q_lora) + return q_lora, q_lora + if not attn._wq_b_native_consumer_checked: + attn._wq_b_native_consumer = _native_mxfp8_consumer(attn.wq_b) + attn._wq_b_native_consumer_checked = True + q_for_wq_b, q_lora = rmsnorm_fake_quant_fp8( + q_lora, + attn.q_norm.weight.data, + attn.q_norm.variance_epsilon, + emit_fp8=attn._wq_b_native_consumer, + ) + return q_lora, q_for_wq_b + + +def input_norm_fake_quant( + layer, hidden_states: torch.Tensor, coefficients=None +) -> Tuple[torch.Tensor, Optional[object]]: + """layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the fp8-grid operand + of its dense projections, or None for non-2-D / non-bf16 rows). coefficients rides in the norm + launch when the fused kernel runs, else it is materialized here.""" + norm = layer.input_layernorm + if not _fake_quant_applies(norm, hidden_states): + if coefficients is not None: + coefficients.materialize() + return norm(hidden_states), None + if not layer._wqkv_a_native_consumer_checked: + # wqkv_a exists only when the q / kv projections are fused + layer._wqkv_a_native_consumer = _native_mxfp8_consumer( + layer.self_attn.wqkv_a if layer.self_attn.fuse_wqa_wkv else None + ) + layer._wqkv_a_native_consumer_checked = True + emit_fp8 = layer._wqkv_a_native_consumer + if coefficients is not None and not coefficients.materialized: + x_quant, hidden_states = rmsnorm_with_sinkhorn( + hidden_states, + norm.weight.data, + norm.variance_epsilon, + coefficients, + emit_fp8=emit_fp8, + ) + return hidden_states, x_quant + x_quant, hidden_states = rmsnorm_fake_quant_fp8( + hidden_states, + norm.weight.data, + norm.variance_epsilon, + emit_fp8=emit_fp8, + ) + return hidden_states, x_quant + + +def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tensor: + """layer.post_attention_layernorm(x); with coefficients (HcCoefficients of the + boundary that produced x) still pending, the reduce + sinkhorn rides in the norm + launch, which then is the Triton row norm rather than the aiter one.""" + norm = layer.post_attention_layernorm + if ( + coefficients is None + or coefficients.materialized + or not _fake_quant_applies(norm, x) + ): + if coefficients is not None: + coefficients.materialize() + return norm(x) + _, out = rmsnorm_with_sinkhorn( + x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False + ) + return out + + +def live_rows(activation, num_tokens: int): + """The first num_tokens rows of a break input; the gfx950 fused q_norm + hands the indexer its Fp8GridActivation wrapper, whose rows live in .x.""" + if isinstance(activation, Fp8GridActivation): + return Fp8GridActivation(activation.x[:num_tokens]) + if isinstance(activation, Mxfp8Activation): + return Mxfp8Activation(activation.q[:num_tokens], activation.scale[:num_tokens]) + return activation[:num_tokens] + + +def wo_b_takes_fp8_grid(attn) -> bool: + """Whether attn.wo_b consumes an Fp8GridActivation, which the wo_a GEMM + then emits from its epilogue; resolved on first use, once the weights are loaded.""" + if attn._wo_b_fp8_grid_operand is None: + quant_method = attn.wo_b.quant_method + attn._wo_b_fp8_grid_operand = bool( + _wo_a_fp8_grid_gemm is not None + and isinstance(quant_method, Fp8LinearMethod) + and quant_method.block_fp8_as_mxfp8 + and attn.wo_b.block_fp8_mxfp8_ready + and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native() + ) + return attn._wo_b_fp8_grid_operand + + +def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool): + """o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched + GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as + an Fp8GridActivation [T, G * R]; None when the fork did not import.""" + if _wo_a_fp8_grid_gemm is None: + return None + # the split-K regime ends at 64 rows, so a request's verify and decode rows would + # take different reduction orders; deterministic inference keeps the single chain + split_k = False if get_exec().deterministic.enable_deterministic_inference else None + y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k) + if fp8_grid: + return Fp8GridActivation(y) + return y.view(o.shape[0], wo_a.shape[0], wo_a.shape[1]) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py new file mode 100644 index 0000000000..cc96490ab4 --- /dev/null +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_hip.py @@ -0,0 +1,247 @@ +"""ROCm glue of deepseek_v4: the branches the model takes under _is_hip, bound there as +_hip. The gfx950 dense fp8 routes live in deepseek_v4_gfx95_dense, the fused mHC +boundary in deepseek_v4_fused_mhc.""" + +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + +from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h +from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled +from sglang.srt.environ import envs +from sglang.srt.layers.attention.hip_flash_mla import ( + hip_attention_needs_head_pad, + resolve_hip_flashmla_backend, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( + is_in_breakable_cuda_graph, +) +from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense +from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module + apply_attention_mhc, + forward_hc_pre_from_prev_fused_boundary, +) +from sglang.srt.runtime_context import get_exec +from sglang.srt.utils import is_gfx95_supported + +live_rows = gfx95_dense.live_rows +wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul + +_is_gfx95_supported = is_gfx95_supported() + + +# ---- MqaAttentionBase / MQALayer ---- + + +def init_mqa_layer(attn, quant_config) -> None: + """The ROCm state of MQALayer.__init__. On the gfx950 32-block route q_norm also emits + the fp8-grid operand of wq_b; whether wq_b consumes native MXFP8, and wo_b the fp8-grid + operand wo_a emits, resolve on first use, once the weights are loaded.""" + attn.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible( + quant_config + ) + attn._wq_b_native_consumer_checked = False + attn._wq_b_native_consumer = False + attn._wo_b_fp8_grid_operand = None + + +def wo_a_emits_fp8_grid(attn) -> bool: + """Whether V4.1's gfx950 wo_a GEMM rounds its output onto wo_b's fp8 grid.""" + return attn.is_dsv41 and gfx95_dense.wo_b_takes_fp8_grid(attn) + + +def wo_a_split_k_allowed() -> bool: + """Whether the V4.1 gfx950 wo_a decode / verify kernels may run: their split-K + reduction order depends on the row count, which batch-invariant and deterministic + inference rule out.""" + return not ( + is_batch_invariant_mode_enabled() + or get_exec().deterministic.enable_deterministic_inference + ) + + +def use_fused_qk_norm_rope(attn) -> bool: + return bool(envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get() and attn.q_head_norm) + + +def q_norm_for_wq_b(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]: + """attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b + consumes); on the gfx950 32-block route the second is already on the fp8 grid.""" + if attn.fused_rmsnorm_fake_quant: + return gfx95_dense.q_norm_fake_quant(attn, q_lora) + q_lora = attn.q_norm(q_lora) + return q_lora, q_lora + + +def fuses_q_rope_into_k_store( + attn, q_out: Optional[torch.Tensor], *, unified: bool, use_cp: bool +) -> bool: + """Whether the K norm-rope-store launch ropes the query heads too: only the plain store + path, where it is the cache writer and the query is consumed in place (no padded copy).""" + return ( + not unified + and not use_cp + and q_out is None + and not attn.q_head_norm + and not envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get() + ) + + +def wq_b_unroped(attn, q) -> torch.Tensor: + """attn._compute_q_b without the RoPE, which the K store launch applies.""" + q, _ = attn.wq_b(q) + return q.view(-1, attn.n_local_heads, attn.head_dim) + + +def compute_q_b( + attn, + q_lora: torch.Tensor, + q_for_wqb, + positions: torch.Tensor, + q_out: Optional[torch.Tensor], + unified: bool, + use_cp: bool, +): + """attn._compute_q_b(q_for_wqb, positions, q_out) as (q, the q_lora the indexer reads, + whether the K norm-rope-store launch ropes q). The indexer takes the fused operand: + its wq_b would re-round onto the same grid.""" + fuse_q_rope = _is_gfx95_supported and fuses_q_rope_into_k_store( + attn, q_out, unified=unified, use_cp=use_cp + ) + if fuse_q_rope: + q = wq_b_unroped(attn, q_for_wqb) + else: + q = attn._compute_q_b(q_for_wqb, positions, q_out) + return q, q_for_wqb, fuse_q_rope + + +def skip_head_pad(attn) -> bool: + # only tilelang is built for padded head widths; aiter and Triton take the real head count + return attn.attn_tp_size > 1 and not hip_attention_needs_head_pad() + + +def attention_inv_rope( + attn, + positions: torch.Tensor, + forward_batch, + unified: bool, + *, + wo_a_applies_inv_rope: bool, +) -> Optional[Tuple[torch.Tensor, torch.Tensor]]: + """(freqs_real, positions) for aiter's gfx950 sparse kernel to apply the inverse RoPE + itself; None for the other kernels, and where the fp8 wo_a front end or the prefill + graph op keeps it.""" + if ( + unified + or not _is_gfx95_supported + or resolve_hip_flashmla_backend() != "aiter_sparse" + or wo_a_applies_inv_rope + ): + return None + if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph(): + return None + return torch.view_as_real(attn.freqs_cis).flatten(-2), positions + + +# ---- DeepseekV4DecoderLayer ---- + + +def init_decoder_layer(layer, quant_config) -> None: + """The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the + pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8 + resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and + the mixing stats; the kernel only supports hc_mult 4.""" + layer.fused_rmsnorm_fp8_quant = gfx95_dense.fused_rmsnorm_fp8_quant_eligible( + quant_config + ) + layer.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible( + quant_config + ) + layer._wqkv_a_native_consumer_checked = False + layer._wqkv_a_native_consumer = False + layer.hc_boundary_fused = ( + _is_gfx95_supported and layer.hc_pre_from_prev_sublayer and layer.hc_mult == 4 + ) + + +def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]: + """V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks, + whose ordinary stores preserve locality for the following mHC reader; None elsewhere.""" + if not ( + _is_gfx95_supported + and layer.config.model_type == "deepseek_v41" + and 768 <= x.shape[0] <= 4096 + and x.shape[1] == 5120 + and residual.shape == (x.shape[0], 4, 5120) + and post.shape == (x.shape[0], 4) + and comb.shape == (x.shape[0], 4, 4) + and x.dtype == residual.dtype == torch.bfloat16 + and post.dtype == comb.dtype == torch.float32 + and all(t.is_contiguous() for t in (x, residual, post, comb)) + ): + return None + return mhc_post_split_h(x, residual, post, comb, block_size=2048) + + +def input_norm( + layer, + hidden_states: torch.Tensor, + allow_aiter_quant: bool, + coefficients, + fused_rmsnorm_fp8_quant, +) -> Tuple[torch.Tensor, Optional[object]]: + """layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the + pre-quantized operand of its dense projections or None). coefficients is the fused + boundary's pending reduce + sinkhorn, hosted by the gfx950 norm launch when that one runs.""" + if layer.fused_rmsnorm_fp8_quant and allow_aiter_quant: + if coefficients is not None: + coefficients.materialize() + x_quant, hidden_states = fused_rmsnorm_fp8_quant( + hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps + ) + return hidden_states, x_quant + if layer.fused_rmsnorm_fake_quant: + return gfx95_dense.input_norm_fake_quant(layer, hidden_states, coefficients) + if coefficients is not None: + coefficients.materialize() + return layer.input_layernorm(hidden_states), None + + +# ---- DeepseekV4Model layer loop ---- + + +def forward_layer_fused_boundary( + model, + i: int, + *, + positions: torch.Tensor, + hidden_states: Optional[torch.Tensor], + input_ids: torch.Tensor, + forward_batch, + input_ids_global: torch.Tensor, + prev_pre: Optional[torch.Tensor], + pending_post: Optional[Tuple[torch.Tensor, ...]], + capture_dspark: bool, +): + """Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next + layer's boundary launch unless that layer reads the residual stream first (Engram gate, + DSpark capture) or there is none; hidden_states is None while a post is pending.""" + nxt = i + 1 + defer_post = ( + nxt < model.end_layer + and model.layers[nxt].engram is None + and not (capture_dspark and nxt in model.dspark_layers_to_capture) + ) + return forward_hc_pre_from_prev_fused_boundary( + model.layers[i], + positions=positions, + hidden_states=hidden_states, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + prev_pre=prev_pre, + pending_post=pending_post, + defer_post=defer_post, + ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 6b60034ca5..dc94bd60b4 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -225,6 +225,13 @@ if _use_aiter_gfx95: if _use_aiter: pass +if _is_hip: + from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_act as _hip_act + from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_moe as _hip_moe +else: + _hip_act = None + _hip_moe = None + if _is_cuda: from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16 elif _is_npu: @@ -401,20 +408,14 @@ class DeepseekV2MLP(nn.Module): ) return down_output + if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked: + _hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2) + if self.use_fused_clamp_act_mul and self.swiglu_limit is not None: from aiter.ops.triton.fusions.fused_clamp_act_mul import ( fused_clamp_act_mul, ) - if not self._fused_clamp_fp8_checked: - from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod - - qm = getattr(self.down_proj, "quant_method", None) - self._fused_clamp_use_fp8 = ( - isinstance(qm, Fp8LinearMethod) and qm.block_quant - ) - self._fused_clamp_fp8_checked = True - if self._fused_clamp_use_fp8: from aiter import dtypes @@ -442,7 +443,9 @@ class DeepseekV2MLP(nn.Module): # Fallback: fused silu+clamp kernel (still faster than unfused) elif self.swiglu_limit is not None: - if _is_npu: + if _is_hip: + x = _hip_act.silu_and_mul_clamp(self, gate_up) + elif _is_npu: x = torch.ops.npu.npu_clipped_swiglu( gate_up, alpha=1, @@ -514,6 +517,10 @@ class MoEGate(nn.Module): hidden_size=config.hidden_size, weight_dtype=self.weight.dtype, ) + # Rows up to which the ROCm split-K router serves the gate (-1: never) + self.rocm_router_max_tokens = ( + _hip_moe.router_max_tokens(self, config, is_hash_moe) if _is_hip else -1 + ) def forward( self, @@ -939,6 +946,9 @@ class DeepseekV2MoE(nn.Module): if forward_batch is not None else None ) + use_vision_topk = self.gate.e_score_correction_bias_vl is not None + if use_vision_topk and _is_hip: + use_vision_topk = _hip_moe.batch_has_images(forward_batch) if not self._enable_a2a_moe: if self._can_dual_stream_graph(hidden_states): fwd = get_forward() @@ -961,6 +971,7 @@ class DeepseekV2MoE(nn.Module): input_ids, input_ids_global=input_ids_global, num_token_non_padded=num_token_non_padded, + use_vision_topk=use_vision_topk, ) else: return self.forward_normal( @@ -970,6 +981,7 @@ class DeepseekV2MoE(nn.Module): input_ids_global=input_ids_global, skip_shared_experts=skip_shared_experts, num_token_non_padded=num_token_non_padded, + use_vision_topk=use_vision_topk, ) else: return self.forward_deepep( @@ -983,6 +995,7 @@ class DeepseekV2MoE(nn.Module): input_ids: Optional[torch.Tensor] = None, input_ids_global: Optional[torch.Tensor] = None, num_token_non_padded: Optional[torch.Tensor] = None, + use_vision_topk: bool = False, ) -> torch.Tensor: # Note(kpham-sgl): issue order satisfies 3 constraints: # - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream; @@ -1016,7 +1029,16 @@ class DeepseekV2MoE(nn.Module): ) # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states, gemm_output_zero_allocator) + if _is_hip: + router_logits, router_logits_partials = _hip_moe.forward_gate( + self, + hidden_states, + gemm_output_zero_allocator, + fused_gate=not use_flashinfer_trtllm_bypass and not use_vision_topk, + ) + else: + router_logits = self.gate(hidden_states, gemm_output_zero_allocator) + router_logits_partials = None if use_flashinfer_trtllm_bypass: topk_output = BypassedTopKOutput( hidden_states=hidden_states, @@ -1029,7 +1051,9 @@ class DeepseekV2MoE(nn.Module): if getattr(self, "is_hash", False) else {} ) - if self.gate.e_score_correction_bias_vl is not None: + if router_logits_partials is not None: + topk_kwargs["router_logits_partials"] = router_logits_partials + if use_vision_topk: topk_output = vision_topk( self, router_logits, @@ -1226,6 +1250,10 @@ class DeepseekV2MoE(nn.Module): ) mhc = current_mhc_post_fusion() + if _is_hip and _hip_moe.fused_all_reduce_mhc( + self, mhc, final_hidden_states + ): + return final_hidden_states if mhc is not None: mhc.start_stats_before_all_reduce() final_hidden_states = post_experts_all_reduce(final_hidden_states) @@ -1243,6 +1271,7 @@ class DeepseekV2MoE(nn.Module): input_ids_global: Optional[torch.Tensor] = None, skip_shared_experts: bool = False, num_token_non_padded: Optional[torch.Tensor] = None, + use_vision_topk: bool = False, ) -> torch.Tensor: if hasattr(self, "shared_experts") and use_intel_amx_backend( self.shared_experts.gate_up_proj @@ -1277,13 +1306,24 @@ class DeepseekV2MoE(nn.Module): pre_quant_input=pre_quant_input, ) # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states, gemm_output_zero_allocator) + if _is_hip: + router_logits, router_logits_partials = _hip_moe.forward_gate( + self, + hidden_states, + gemm_output_zero_allocator, + fused_gate=not use_vision_topk, + ) + else: + router_logits = self.gate(hidden_states, gemm_output_zero_allocator) + router_logits_partials = None topk_kwargs = ( {"input_ids": input_ids_global} if getattr(self, "is_hash", False) else {} ) - if self.gate.e_score_correction_bias_vl is not None: + if router_logits_partials is not None: + topk_kwargs["router_logits_partials"] = router_logits_partials + if use_vision_topk: topk_output = vision_topk( self, router_logits, @@ -1382,6 +1422,10 @@ class DeepseekV2MoE(nn.Module): from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion mhc = current_mhc_post_fusion() + if _is_hip and _hip_moe.fused_all_reduce_mhc( + self, mhc, final_hidden_states + ): + return final_hidden_states if mhc is not None: mhc.start_stats_before_all_reduce() final_hidden_states = post_experts_all_reduce(final_hidden_states) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 2413876c61..0e6543c434 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -425,7 +425,9 @@ _wo_a_aiter_batched_gemm_disabled = False _wo_a_fp8_mxscale = None _wo_a_fp8_mxscale_fused_invrope = None _wo_a_weight_scale_to_e8m0 = None +_hip = None if _is_hip: + from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import ( apply_wo_a_fp8_mxscale, apply_wo_a_fp8_mxscale_fused_invrope, @@ -459,31 +461,48 @@ def _apply_wo_a_bf16_matmul( fuse_mxfp8_quant: bool = False, is_prefill: bool = False, fast_path: bool = False, + fp8_grid: bool = False, ) -> torch.Tensor | Mxfp8SwizzledInput: # o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated # on the exact validated TP4 shapes and write token-major output directly. global _wo_a_aiter_batched_gemm_disabled - if ( - fast_path - and _is_cuda + # the gfx950 routes, like the CUDA ones, are V4.1's (fast_path); DSv4 keeps the + # aiter batched GEMM / einsum below + hip_fast_path = fast_path and _is_hip and _is_gfx95_supported + hip_decode_verify = ( + hip_fast_path and ( - ( - is_decode - and o.shape[0] == 1 - and (get_platform().is_blackwell or get_platform().is_sm90) - ) - or ( - is_target_verify - and 0 < o.shape[0] <= 384 - and get_platform().is_blackwell - ) - or ( - is_prefill - and 4096 <= o.shape[0] <= 65536 - and get_platform().is_blackwell + (is_decode and o.shape[0] == 1 and o.is_contiguous()) + or (is_target_verify and 2 <= o.shape[0] <= 384) + ) + and _hip.wo_a_split_k_allowed() + ) + if ( + ( + fast_path + and _is_cuda + and ( + ( + is_decode + and o.shape[0] == 1 + and (get_platform().is_blackwell or get_platform().is_sm90) + ) + or ( + is_target_verify + and 0 < o.shape[0] <= 384 + and get_platform().is_blackwell + ) + or ( + is_prefill + and 4096 <= o.shape[0] <= 65536 + and get_platform().is_blackwell + ) ) ) - and o.shape[1:] == (2, 4096) + or hip_decode_verify + or (hip_fast_path and is_prefill and 4096 <= o.shape[0] <= 65536) + ) and ( + o.shape[1:] == (2, 4096) and wo_a.shape == (2, 1024, 4096) and o.dtype == wo_a.dtype == torch.bfloat16 and o.stride(2) == 1 @@ -507,6 +526,11 @@ def _apply_wo_a_bf16_matmul( o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1) ) return result + # the 16-row decode kernels also serve V4.1's target-verify rows + if (is_decode or is_target_verify) and hip_fast_path: + y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid) + if y is not None: + return y if ( is_decode and _wo_a_aiter_batched_gemm_enabled @@ -722,7 +746,11 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None: get_attn_backend().forward_low_ratio_sources( layer=layer, x=x[:real_num_tokens], - q_lora=q_lora[:real_num_tokens], + q_lora=( + _hip.live_rows(q_lora, real_num_tokens) + if _is_hip + else q_lora[:real_num_tokens] + ), positions=positions[:real_num_tokens], forward_batch=forward_batch, ) @@ -1102,7 +1130,11 @@ class MQALayer(MqaAttentionBase): self.register_buffer("sin_cache", sin_cache, persistent=False) if alt_streams is not None and ( - (_is_cuda and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get()) + ( + # the gfx950 overlap is validated on V4.1 only; DSv4 on ROCm keeps one stream + (_is_cuda or (_is_gfx95_supported and self.is_dsv41)) + and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get() + ) or (_is_npu and envs.SGLANG_NPU_USE_MULTI_STREAM.get()) ): self.alt_streams = alt_streams[:3] @@ -1173,9 +1205,7 @@ class MQALayer(MqaAttentionBase): prefix=add_prefix("attn_mqa", prefix), ) - self.use_fused_qk_norm_rope = ( - _is_hip and envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get() - ) + self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self) # Static eligibility; token count and wo_b's output format are checked # in forward, after weights have loaded. self.use_fused_wo_a = ( @@ -1188,6 +1218,8 @@ class MQALayer(MqaAttentionBase): and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096) and (self.n_local_groups, self.o_lora_rank) == (2, 1024) ) + if _is_hip: + _hip.init_mqa_layer(self, quant_config) # KV cache write is always fused into the K kernel # (`_compute_kv_to_cache`), so the legacy "overlap store cache" flag @@ -1256,6 +1288,8 @@ class MQALayer(MqaAttentionBase): self, q: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor | Mxfp8SwizzledInput]: # The indexer needs the BF16 normalized row; wq_b needs the quantized one. + if _is_hip: + return _hip.q_norm_for_wq_b(self, q) method = self.wq_b.quant_method if ( _is_cuda @@ -1310,13 +1344,13 @@ class MQALayer(MqaAttentionBase): q = q.view(-1, self.n_local_heads, self.head_dim) if not self.q_head_norm: if ( - _is_cuda + (_is_cuda or _is_gfx95_supported) and q_out is not None and ( 0 < q.shape[0] <= 8 or ( self.is_dsv41 - and get_platform().is_blackwell + and (get_platform().is_blackwell or _is_gfx95_supported) and self.n_local_heads == 16 and 4096 <= q.shape[0] <= 65536 ) @@ -1354,13 +1388,16 @@ class MQALayer(MqaAttentionBase): forward_batch: ForwardBatch, attn_backend, qkv_a: Optional[torch.Tensor] = None, + q_rope: Optional[torch.Tensor] = None, ) -> None: """Fused: rmsnorm + RoPE + write directly to FlashMLA paged cache. Replaces the bf16-kv-intermediate path. Used everywhere except the DSA prefill-CP case (which needs bf16 kv for the cross-rank all-gather). + q_rope ([T, H, head_dim]) has its query heads roped by the same launch. """ if envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get(): + assert q_rope is None # Quantize the nope payload from bf16-rounded values (the fused # kernel quantizes from fp32 registers; the bf16 rounding moves # values across fp8 bins relative to bf16-sourced consumers). @@ -1384,6 +1421,8 @@ class MQALayer(MqaAttentionBase): eps=self.eps, freqs_cis=self.freqs_cis, positions=positions, + # the query RoPE fused into this store is HIP only + **({} if q_rope is None else {"q": q_rope}), ) def _compute_kv_bf16( @@ -1724,8 +1763,8 @@ class MQALayer(MqaAttentionBase): dtype=x.dtype, ) else: - q_lora = self.q_norm(q_lora) - q = self._compute_q_b(q_lora, positions, q_out) + q_lora, q_for_wqb = self._normalize_q_lora(q_lora) + q = self._compute_q_b(q_for_wqb, positions, q_out) self._compute_kv_to_cache( x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a ) @@ -1982,7 +2021,13 @@ class MQALayer(MqaAttentionBase): q_out.copy_(q) else: q_lora, q_for_wqb = self._normalize_q_lora(q_lora) - q = self._compute_q_b(q_for_wqb, positions, q_out) + if _is_hip: + q, q_lora, fuse_q_rope = _hip.compute_q_b( + self, q_lora, q_for_wqb, positions, q_out, unified, use_cp + ) + else: + q = self._compute_q_b(q_for_wqb, positions, q_out) + fuse_q_rope = False if unified: # unified_kv prefill: keep bf16 kv; the backend writes # the ring AFTER attention (2-source path). @@ -2028,7 +2073,12 @@ class MQALayer(MqaAttentionBase): kv = None else: self._compute_kv_to_cache( - x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a + x_linear, + positions, + forward_batch, + attn_backend, + qkv_a=qkv_a, + q_rope=q if fuse_q_rope else None, ) kv = None @@ -2132,8 +2182,7 @@ class MQALayer(MqaAttentionBase): ) low_ratio_multi_stream = ( - _is_cuda - and get_platform().is_blackwell + ((_is_cuda and get_platform().is_blackwell) or _is_gfx95_supported) and self.compress_ratio in (1, 2) and self.alt_streams is not None and ( @@ -2141,8 +2190,11 @@ class MQALayer(MqaAttentionBase): or ( forward_batch.forward_mode.is_target_verify() # Other MXFP8 backends may share mutable GEMM workspace. - and getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None) - == Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL + and ( + _is_gfx95_supported + or getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None) + == Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL + ) ) ) ) @@ -2223,7 +2275,11 @@ class MQALayer(MqaAttentionBase): k_rope = rope_pool.new_empty((x.shape[0], rope_pool.shape[-1])) kernel_num_heads = self.n_local_heads else: - kernel_num_heads = self._kernel_num_heads(x.shape[0]) + kernel_num_heads = ( + self.n_local_heads + if _is_hip and not unified and _hip.skip_head_pad(self) + else self._kernel_num_heads(x.shape[0]) + ) if kernel_num_heads != self.n_local_heads: # Backends without an exact-head specialization retain the existing # padded shape. attn_sink is sliced to this rank and padded to match. @@ -2318,8 +2374,21 @@ class MQALayer(MqaAttentionBase): # its normal causally-indexed store from attn_k = kv. attn_k = kv if kv is not None else q + # HIP: the attention kernel may apply the inverse RoPE itself; the else-branch below keeps it + inv_rope = ( + _hip.attention_inv_rope( + self, + positions, + forward_batch, + unified, + wo_a_applies_inv_rope=self.wo_a_fp8 + and _wo_a_fp8_mxscale_fused_invrope is not None, + ) + if _is_hip + else None + ) if unified: - # only the HIP radix backend takes these two; passing them always would + # only the HIP radix backend takes these; passing them always would # leave non-ROCm depending on the **_ in its forward() to drop them, and # no test on that side would notice if the **_ went away rope_kwargs = {} @@ -2327,6 +2396,8 @@ class MQALayer(MqaAttentionBase): rope_kwargs["q_rope"] = q_rope if k_rope is not None: rope_kwargs["k_rope"] = k_rope + if inv_rope is not None: + rope_kwargs["inv_rope"] = inv_rope o = attn_backend.forward( q=q_out if q_out is not None else q, k=attn_k, @@ -2364,6 +2435,8 @@ class MQALayer(MqaAttentionBase): compress_ratio=self.compress_ratio, attn_sink=attn_sink, save_kv_cache=save_kv_cache, + # HIP only, for the same reason as rope_kwargs above + **({} if inv_rope is None else {"inv_rope": inv_rope}), ) o = o[:, tp_slice, :] if ( @@ -2405,7 +2478,7 @@ class MQALayer(MqaAttentionBase): sin4, qk_nope_dim=self.qk_nope_head_dim, ) - elif not fuse_rope_wo_a: + elif inv_rope is None and not fuse_rope_wo_a: # The fused path folds this in; it must not run twice. fused_rope_inplace( o[..., -self.qk_rope_head_dim :], @@ -2510,6 +2583,7 @@ class MQALayer(MqaAttentionBase): is_prefill=forward_batch.forward_mode.is_extend_without_speculative(), fast_path=self.is_dsv41, fuse_mxfp8_quant=fuse_mxfp8_quant, + fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self), ) else: o = _apply_gguf_grouped_wo_a( @@ -2519,16 +2593,21 @@ class MQALayer(MqaAttentionBase): self.o_lora_rank, ) + return self._project_wo_b(o) + + def _project_wo_b(self, o): from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion mhc = current_mhc_post_fusion() o, _ = self.wo_b( - o if isinstance(o, Mxfp8SwizzledInput) else o.flatten(1), + o.flatten(1) if isinstance(o, torch.Tensor) else o, skip_all_reduce=mhc is not None, ) if mhc is not None and mhc.overlap_only: mhc.start_stats_before_all_reduce() o = attn_tp_all_reduce(o) + elif mhc is not None and _is_hip: + _hip.apply_attention_mhc(o, mhc) elif mhc is not None: from sglang.kernels.ops.communication.all_reduce_mhc import ( all_reduce_mhc_norm, @@ -2679,6 +2758,8 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_pre_from_prev_sublayer = config.hc_pre_from_prev_sublayer if self.hc_pre_from_prev_sublayer: self.use_fused_mhc_post_pre = False + if _is_hip: + _hip.init_decoder_layer(self, quant_config) self.engram = None if engram_layout is not None and layer_id in engram_layout.layer_ids: self.engram = Engram( @@ -2759,6 +2840,25 @@ class DeepseekV4DecoderLayer(nn.Module): ) self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data) + def _input_norm( + self, + hidden_states: torch.Tensor, + allow_aiter_quant: bool = True, + coefficients=None, + ) -> Tuple[torch.Tensor, Optional[Tuple]]: + """input_layernorm(hidden_states) as (the bf16 norm attention reads, the pre-quantized + operand of its dense projections or None); coefficients is a ROCm boundary's pending + reduce + sinkhorn.""" + if _is_hip: + return _hip.input_norm( + self, + hidden_states, + allow_aiter_quant, + coefficients, + _fused_rmsnorm_fp8_quant, + ) + return self.input_layernorm(hidden_states), None + def hc_pre( self, x: torch.Tensor, @@ -2959,6 +3059,11 @@ class DeepseekV4DecoderLayer(nn.Module): ): return mhc_post_split_h(x, residual, post, comb) + if _is_hip: + y = _hip.hc_post(self, x, residual, post, comb) + if y is not None: + return y + if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get(): from flashinfer.mhc import mhc_post @@ -3053,15 +3158,9 @@ class DeepseekV4DecoderLayer(nn.Module): # Triton fused post+pre (gfx95 small-batch or gfx1250) returns # norm_fused=False — the input layernorm is NOT folded. # gfx95 takes the fp8-quant path; gfx1250 takes plain layernorm. - if _use_aiter and _is_gfx95_supported: - x_quant, hidden_states = _fused_rmsnorm_fp8_quant( - hidden_states, - self.input_layernorm.weight, - self.rms_norm_eps, - ) - else: - hidden_states = self.input_layernorm(hidden_states) - x_quant = None + hidden_states, x_quant = self._input_norm( + hidden_states, allow_aiter_quant=_is_gfx95_supported + ) else: x_quant = None else: @@ -3078,15 +3177,10 @@ class DeepseekV4DecoderLayer(nn.Module): forward_batch=forward_batch, ) if not norm_fused: - if _use_aiter and _is_gfx95_supported: - x_quant, hidden_states = _fused_rmsnorm_fp8_quant( - hidden_states, - self.input_layernorm.weight, - self.rms_norm_eps, - ) - else: - hidden_states = self.input_layernorm(hidden_states) - x_quant = None + # gfx1250 takes plain layernorm here + hidden_states, x_quant = self._input_norm( + hidden_states, allow_aiter_quant=_is_gfx95_supported + ) else: x_quant = None else: @@ -3100,15 +3194,10 @@ class DeepseekV4DecoderLayer(nn.Module): forward_batch=forward_batch, ) if not norm_fused: - if _use_aiter and _is_gfx95_supported: - x_quant, hidden_states = _fused_rmsnorm_fp8_quant( - hidden_states, - self.input_layernorm.weight, - self.rms_norm_eps, - ) - else: - hidden_states = self.input_layernorm(hidden_states) - x_quant = None + # gfx1250 takes plain layernorm here + hidden_states, x_quant = self._input_norm( + hidden_states, allow_aiter_quant=_is_gfx95_supported + ) else: x_quant = None @@ -3290,10 +3379,16 @@ class DeepseekV4DecoderLayer(nn.Module): if ( x.is_cuda - and torch.version.cuda is not None and ( - get_platform().is_blackwell - or (get_platform().is_sm90 and x.shape[0] == 1) + # gfx950: the fused Triton port wins at every row count (MI350X) + _is_gfx95_supported + or ( + torch.version.cuda is not None + and ( + get_platform().is_blackwell + or (get_platform().is_sm90 and x.shape[0] == 1) + ) + ) ) and x.dtype == torch.bfloat16 ): @@ -3944,15 +4039,7 @@ class DeepseekV4DecoderLayer(nn.Module): forward_batch=forward_batch, ) if not norm_fused: - if _use_aiter and (_is_gfx95_supported or _is_gfx1250_supported): - x_quant, hidden_states = _fused_rmsnorm_fp8_quant( - hidden_states, - self.input_layernorm.weight, - self.rms_norm_eps, - ) - else: - hidden_states = self.input_layernorm(hidden_states) - x_quant = None + hidden_states, x_quant = self._input_norm(hidden_states) else: x_quant = None @@ -4433,14 +4520,20 @@ class DeepseekV4Model(nn.Module): precomputed_attn = None combined_attn = None normalized_attn = None + # HIP: the fused boundary hands the next layer its FFN hc_post unapplied + pending_post = None for i in range(self.start_layer, self.end_layer): if tail is not None and i == self.late_layer_start: combined_attn = None normalized_attn = None # Decode reaches back at most SWA_WINDOW positions. saved_full = attn_backend.enter_late_layer_tail(forward_batch) - hidden_states, prev_pre, input_ids, input_ids_global = ( - tail.rows(hidden_states), + if pending_post is not None: + # the fused boundary carries the residual stream here, not in hidden_states + pending_post = tuple(tail.rows(t) for t in pending_post) + else: + hidden_states = tail.rows(hidden_states) + prev_pre, input_ids, input_ids_global = ( tail.rows(prev_pre), tail.rows(input_ids), tail.rows(input_ids_global), @@ -4484,10 +4577,13 @@ class DeepseekV4Model(nn.Module): next_input = [] # The next layer can consume a collapsed input only if no Engram # or row selection changes the residual between the two layers. + # The HIP fused boundary carries the residual in pending_post and + # leaves hidden_states None; it never consumes a collapsed input. next_combined = ( [] if ( - self.config.model_type == "deepseek_v41" + hidden_states is not None + and self.config.model_type == "deepseek_v41" and ( 128 <= hidden_states.shape[0] <= 384 or ( @@ -4505,10 +4601,10 @@ class DeepseekV4Model(nn.Module): next_norm = self.layers[i + 1].input_layernorm if ( self.config.model_type == "deepseek_v41" + and get_platform().is_blackwell and i + 1 < self.end_layer and tail is None and hidden_states.is_cuda - and get_platform().is_blackwell and 0 < hidden_states.shape[0] <= 8 and ( forward_batch.forward_mode.is_decode() @@ -4533,6 +4629,24 @@ class DeepseekV4Model(nn.Module): ): next_norm = norm with ctx: + if _is_hip and self.layers[i].hc_boundary_fused: + hidden_states, prev_pre, pending_post = ( + _hip.forward_layer_fused_boundary( + self, + i, + positions=positions, + hidden_states=hidden_states, + input_ids=input_ids, + forward_batch=forward_batch, + input_ids_global=input_ids_global, + prev_pre=prev_pre, + pending_post=pending_post, + capture_dspark=capture_dspark, + ) + ) + # the fused boundary never publishes a collapsed input for the next layer + precomputed_attn = combined_attn = normalized_attn = None + continue hidden_states, prev_pre = self.layers[i].forward_hc_pre_from_prev( positions=positions, hidden_states=hidden_states, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index e0a5596f5a..4ba8f59e13 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -335,10 +335,12 @@ class DSparkWorkerV2(BaseSpecWorker): and not get_parallel().enable_dp_attention and self.model_runner.pp_size == 1 ) + # ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts + # need the group's pynccl communicator if ( (self._verify_planner.is_compact_mode or static_epilogue_supported) and self._decode_graph_allowed - and is_cuda() + and (is_cuda() or (is_cuda_alike() and target_is_dsv41)) ): self._verify_epilogue = DsparkVerifyEpilogue( max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), diff --git a/test/registered/amd/test_dsv4_hip_bcg_metadata.py b/test/registered/amd/test_dsv4_hip_bcg_metadata.py index 44210a77cb..cb8bec3f1c 100644 --- a/test/registered/amd/test_dsv4_hip_bcg_metadata.py +++ b/test/registered/amd/test_dsv4_hip_bcg_metadata.py @@ -13,10 +13,91 @@ from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( UnifiedKvMetadata, _match_num_queries, ) +from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase -register_amd_ci(est_time=5, suite="stage-b-test-1-gpu-small-amd-mi35x") +register_amd_ci(est_time=20, suite="stage-b-test-1-gpu-small-amd-mi35x") + +SWA_WINDOW = 128 +PAGE_SIZE = 256 +NUM_REQ_SLOTS = 6 +MAX_CONTEXT = 512 +# Block slots (out_cache_loc) live past every req_to_token slot. +OUT_LOC_BASE = NUM_REQ_SLOTS * MAX_CONTEXT + 1 +NUM_FULL_SLOTS = OUT_LOC_BASE + 4096 + + +def _make_backend(*, block_size, device, is_dspark_draft=True, low_ratios=()): + backend = object.__new__(DeepseekV4HipRadixBackend) + backend.device = device + backend.cuda_int32_kwargs = {"device": device, "dtype": torch.int32} + backend.swa_page_size = SWA_WINDOW + backend.page_size = PAGE_SIZE + g = torch.Generator().manual_seed(0) + # Distinct full slots per (request slot, position); slot 0 stays the dummy. + req_to_token = ( + torch.randperm(NUM_REQ_SLOTS * MAX_CONTEXT, generator=g).view( + NUM_REQ_SLOTS, MAX_CONTEXT + ) + + 1 + ) + backend.req_to_token = req_to_token.to(device=device, dtype=torch.int32) + backend.req_to_token_pool = SimpleNamespace(req_to_token=backend.req_to_token) + backend.MAX_SEQ_LEN_FOR_CAPTURE = MAX_CONTEXT + backend.max_context_len = MAX_CONTEXT + # full -> swa: injective and not the identity, so a wrong source shows. + full_to_swa = (torch.arange(NUM_FULL_SLOTS) * 3 + 7).to( + device=device, dtype=torch.int64 + ) + backend.token_to_kv_pool = SimpleNamespace( + full_to_swa_index_mapping=full_to_swa, + translate_loc_from_full_to_swa=lambda idx: full_to_swa[idx.to(torch.int64)], + unified_swa_pages=0, + request_window=None, + ) + backend.index_topk = 512 + backend.present_ratios = tuple(sorted(low_ratios)) + backend.low_ratios = tuple(low_ratios) + backend.has_c4 = False + backend.has_c128 = False + backend.candidate_masks = None + backend.low_ratio_identity_skip = False + backend.low_ratio_candidate_span = None + backend.enable_deepseek_v4_fp4_indexer = False + backend.topk = 1 + backend.mtp_enabled = True + backend.speculative_num_steps = 0 + backend.speculative_step_id = 0 + backend.speculative_num_draft_tokens = block_size + 1 + backend.is_dspark = True + backend.needs_cpu_seq_lens = False + backend._fp4_graph_row_limit = None + backend.is_draft_worker = is_dspark_draft + # compressed-KV metadata is built only on the target worker + backend.need_compress = not is_dspark_draft + backend.is_dspark_draft = is_dspark_draft + backend.is_dsv41 = True + # The draft verifies gamma rows, the target gamma + 1 (bonus included). + backend.target_verify_num_draft_tokens = ( + block_size if is_dspark_draft else block_size + 1 + ) + backend.forward_metadata = None + return backend + + +def _expected_block_row(backend, *, req_slot, prefix, out_loc_block, width): + """Reference window row: the last min(prefix, SWA_WINDOW) committed tokens, + then the block's slots, -1 padded; all through the full -> swa map.""" + full_to_swa = backend.token_to_kv_pool.full_to_swa_index_mapping + ctx = min(prefix, SWA_WINDOW) + row = torch.full((width,), -1, dtype=torch.int32, device=full_to_swa.device) + if ctx: + ctx_full = backend.req_to_token[req_slot, prefix - ctx : prefix].to(torch.int64) + row[:ctx] = full_to_swa[ctx_full].to(torch.int32) + row[ctx : ctx + out_loc_block.numel()] = full_to_swa[out_loc_block].to(torch.int32) + return row, ctx @unittest.skipUnless(is_hip(), "DeepSeek V4 HIP radix backend requires ROCm") @@ -75,6 +156,9 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): def test_unified_prefill_metadata_pads_to_capture_bucket(self): backend = object.__new__(DeepseekV4HipRadixBackend) + backend.low_ratios = () + backend.has_c4 = True + backend.has_c128 = True backend.token_to_kv_pool = SimpleNamespace(unified_swa_window=128) core = self._make_core_metadata(0) core.positions_casual = torch.tensor([0, 1, 2, 0], dtype=torch.int32) @@ -111,6 +195,9 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): def test_eager_prefill_marks_host_proven_token_count_exact(self): backend = object.__new__(DeepseekV4HipRadixBackend) + backend.low_ratios = () + backend.has_c4 = True + backend.has_c128 = True backend.req_to_token = torch.zeros((2, 8), dtype=torch.int32) backend.token_to_kv_pool = object() core = self._make_core_metadata(0) @@ -159,6 +246,9 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): def test_prefill_bcg_uses_bucket_sized_gpu_compressor_plans(self): backend = object.__new__(DeepseekV4HipRadixBackend) + backend.low_ratios = () + backend.has_c4 = True + backend.has_c128 = True backend.req_to_token = torch.zeros((2, 8), dtype=torch.int32) backend.token_to_kv_pool = object() core = self._make_core_metadata(0) @@ -222,6 +312,9 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): def test_capture_builds_graph_compatible_metadata_and_workspace(self): capture_metadata = DSV4Metadata(object(), indexer_metadata=None) backend = object.__new__(DeepseekV4HipRadixBackend) + backend.low_ratios = () + backend.has_c4 = True + backend.has_c128 = True backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 backend._build_forward_metadata = mock.Mock(return_value=capture_metadata) backend.init_forward_metadata_in_graph = mock.Mock() @@ -348,6 +441,9 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): capture_metadata.refresh_for_breakable_cuda_graph_replay_ = mock.Mock() backend = object.__new__(DeepseekV4HipRadixBackend) + backend.low_ratios = () + backend.has_c4 = True + backend.has_c128 = True backend.MAX_SEQ_LEN_FOR_CAPTURE = 4096 backend._build_forward_metadata = mock.Mock(return_value=replay_metadata) backend.init_forward_metadata_in_graph = mock.Mock() @@ -415,5 +511,269 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): ].prepare_forward_metadata_for_breakable_cuda_graph_replay.assert_not_called() +@unittest.skipUnless(is_hip(), "HIP DeepSeek-V4 backend") +class TestDsparkDraftBlockWindowHip(CustomTestCase): + def setUp(self): + self.device = torch.device("cuda") + + def test_graph_capture_and_replay_route_the_draft_through_the_block_window(self): + """A TARGET_VERIFY capture on the DSpark draft must build the block window, and + a replay must refresh it in place for new lengths and slots, whether or not the + replay batch carries a CPU mirror of the lengths (the draft-window bucket must + not read seq_lens_cpu).""" + for cpu_mirror in (True, False): + with self.subTest(cpu_mirror=cpu_mirror): + self._check_block_window_replay(cpu_mirror=cpu_mirror) + + def test_dsv4_draft_keeps_the_causal_verify_indices(self): + """Only V4.1's DSpark draft takes the block window; DSv4's (V4-Pro) keeps main's.""" + backend = _make_backend(block_size=3, device=self.device) + self.assertTrue(backend._uses_dspark_draft_window()) + backend.is_dsv41 = False + self.assertFalse(backend._uses_dspark_draft_window()) + + def _check_block_window_replay(self, *, cpu_mirror): + block = 3 + bs = 2 + backend = _make_backend(block_size=block, device=self.device) + backend.init_cuda_graph_state(max_bs=bs, max_num_tokens=bs * block) + + def make_batch(prefix_list, out_base, cpu_extended): + prefix = torch.tensor(prefix_list, dtype=torch.int32, device=self.device) + out_loc = ( + torch.arange(bs * block, device=self.device, dtype=torch.int64) + + out_base + ) + seq_lens_cpu = torch.tensor(prefix_list, dtype=torch.int64) + if cpu_extended: + # The worker hands the draft a seq_lens_cpu already + block. + seq_lens_cpu = seq_lens_cpu + block + return SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=bs, + req_pool_indices=torch.tensor( + [4, 1], dtype=torch.int32, device=self.device + ), + seq_lens=prefix, + seq_lens_cpu=seq_lens_cpu, + seq_lens_sum=int(seq_lens_cpu.sum()), + positions=torch.zeros( + bs * block, dtype=torch.int64, device=self.device + ), + out_cache_loc=out_loc, + spec_info=SimpleNamespace(draft_token_num=block), + ) + + capture_batch = make_batch([1, 1], OUT_LOC_BASE, cpu_extended=False) + backend.init_forward_metadata_out_graph(capture_batch, in_capture=True) + captured = backend.forward_metadata + core = captured.core_attn_metadata + width = core.swa_page_indices.shape[1] + + replay_batch = make_batch([150, 7], OUT_LOC_BASE + 100, cpu_extended=True) + if not cpu_mirror: + replay_batch.seq_lens_cpu = None + replay_batch.seq_lens_sum = None + backend.init_forward_metadata_out_graph(replay_batch) + self.assertIs(backend.forward_metadata, captured) + for b, p in enumerate([150, 7]): + row, ctx = _expected_block_row( + backend, + req_slot=int(replay_batch.req_pool_indices[b]), + prefix=p, + out_loc_block=replay_batch.out_cache_loc[b * block : (b + 1) * block], + width=width, + ) + for j in range(block): + r = b * block + j + self.assertTrue(torch.equal(core.swa_page_indices[r], row), (b, j)) + self.assertEqual(int(core.swa_topk_lengths[r]), ctx + block) + # Derived from the device prefix, not the pre-extended CPU copy. + self.assertEqual(int(core.positions_casual[r]), p + j) + + +@unittest.skipUnless(is_hip(), "HIP DeepSeek-V4 backend") +class TestAiterSparseLengthFoldPerStep(CustomTestCase): + """The length-fold cache must not outlive the forward: a TARGET_VERIFY bucket keeps + one core object across steps, so the warmup's lists would be replayed.""" + + def test_graph_bucket_refold_follows_the_new_lengths(self): + from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( + _fold_lengths_for_aiter_sparse, + ) + + device = torch.device("cuda") + block, bs = 3, 2 + backend = _make_backend(block_size=block, device=device) + backend.init_cuda_graph_state(max_bs=bs, max_num_tokens=bs * block) + + def make_batch(prefix_list, out_base): + prefix = torch.tensor(prefix_list, dtype=torch.int32, device=device) + seq_lens_cpu = torch.tensor(prefix_list, dtype=torch.int64) + return SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=bs, + req_pool_indices=torch.tensor([4, 1], dtype=torch.int32, device=device), + seq_lens=prefix, + seq_lens_cpu=seq_lens_cpu, + seq_lens_sum=int(seq_lens_cpu.sum()), + positions=torch.zeros(bs * block, dtype=torch.int64, device=device), + out_cache_loc=torch.arange(bs * block, device=device, dtype=torch.int64) + + out_base, + spec_info=SimpleNamespace(draft_token_num=block), + ) + + def fold(core): + masked, _ = _fold_lengths_for_aiter_sparse( + core, + 0, + core.swa_page_indices.unsqueeze(1), + core.swa_topk_lengths, + None, + None, + ) + return masked.squeeze(1) + + # Warmup-like forward at the capture lengths (prefix 1: 1 + block valid). + backend.init_forward_metadata_out_graph( + make_batch([1, 1], OUT_LOC_BASE), in_capture=True + ) + core = backend.forward_metadata.core_attn_metadata + warm = fold(core).clone() + self.assertTrue( + torch.equal( + (warm >= 0).sum(1), torch.full((bs * block,), 1 + block, device=device) + ) + ) + + # Next step: replay copies longer contexts into the same bucket object. + replay = make_batch([150, 7], OUT_LOC_BASE + 100) + backend.init_forward_metadata_out_graph(replay) + self.assertIs(backend.forward_metadata.core_attn_metadata, core) + backend.init_forward_metadata_in_graph(replay) + got = fold(core) + expect_valid = torch.tensor( + [SWA_WINDOW + block] * block + [7 + block] * block, device=device + ) + self.assertTrue(torch.equal((got >= 0).sum(1), expect_valid)) + self.assertTrue( + torch.equal( + got, + core.swa_page_indices.clone().masked_fill( + torch.arange(got.shape[1], device=device)[None, :] + >= core.swa_topk_lengths[:, None], + -1, + ), + ) + ) + self.assertFalse(torch.equal(got, warm)) + + +@unittest.skipUnless(is_hip(), "HIP DeepSeek-V4 backend") +class TestLowRatioTargetVerifyHip(CustomTestCase): + def setUp(self): + self.device = torch.device("cuda") + import sglang.srt.layers.attention.deepseek_v4_backend_hip_radix as module + + self.module = module + + def test_target_verify_device_lengths_match_cpu_and_replay(self): + backend = _make_backend(block_size=4, device=self.device, is_dspark_draft=False) + backend.has_c128 = True + backend.token_to_kv_pool.swa_page_size = SWA_WINDOW + backend.token_to_kv_pool._unified_kv = False + backend.token_to_kv_pool.get_ring_size = lambda **kwargs: 128 + prefix = torch.tensor([125, 255], dtype=torch.int32, device=self.device) + slots = torch.tensor([1, 4], dtype=torch.int32, device=self.device) + count = 2 * backend.target_verify_num_draft_tokens + out_loc = torch.arange(count, device=self.device) + OUT_LOC_BASE + + def build(cpu_lengths): + return backend.init_forward_metadata_target_verify_old( + max_seq_len=MAX_CONTEXT - backend.target_verify_num_draft_tokens, + req_pool_indices=slots, + seq_lens=prefix, + seq_lens_cpu=cpu_lengths, + out_cache_loc=out_loc, + use_prefill_cuda_graph=True, + ) + + # Build the full compressor plan on device, not just the raw wrapper. + build(None) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = build(None) + for lengths, request_slots in (([125, 255], [1, 4]), ([254, 381], [4, 2])): + prefix.copy_(torch.tensor(lengths, device=self.device)) + slots.copy_(torch.tensor(request_slots, device=self.device)) + out_loc.add_(count) + graph.replay() + reference = build(lengths) + for name in ("seq_lens_casual", "swa_page_indices", "swa_topk_lengths"): + self.assertTrue( + torch.equal( + getattr(captured.core_metadata, name), + getattr(reference.core_metadata, name), + ), + name, + ) + for name in ("plan_c", "plan_w"): + self.assertTrue( + torch.equal( + getattr(captured.c128_compress_metadata, name), + getattr(reference.c128_compress_metadata, name), + ), + name, + ) + + +@unittest.skipUnless(is_hip(), "HIP DeepSeek-V4 backend") +class TestEncoderReplayRequestWindowHip(CustomTestCase): + """Encoder SWA bounded replay floors each request's window at its first row, + the floor the CUDA backend passes to window_layout.""" + + class _Stop(Exception): + pass + + def _floor_passed(self, *, encoder_replay, swa_replay_start=None): + backend = object.__new__(DeepseekV4HipRadixBackend) + backend.token_to_kv_pool = SimpleNamespace( + request_window=SimpleNamespace(capacity=256) + ) + backend.encoder_replay = encoder_replay + seen = {} + + def fake_window_layout(req, pos, **kwargs): + seen.update(kwargs) + raise self._Stop + + with mock.patch( + "sglang.srt.layers.attention.deepseek_v4_backend_hip_radix.window_layout", + side_effect=fake_window_layout, + ): + with self.assertRaises(self._Stop): + backend.make_core_attn_metadata( + req_to_token=None, + req_pool_indices_repeated=torch.tensor([3, 3, 3, 5, 5]), + seq_lens_casual=torch.tensor([10, 11, 12, 20, 21]), + max_seq_len=32, + out_loc=None, + swa_replay_start=swa_replay_start, + ) + self.assertNotIn("replay", seen) + return seen["floor"] + + def test_replay_floors_each_request_at_its_first_row(self): + floor = self._floor_passed(encoder_replay=True) + self.assertEqual(floor.tolist(), [9, 9, 9, 19, 19]) + + def test_without_replay_the_caller_floor_passes_through(self): + start = torch.tensor([4, 4, 4, 7, 7]) + self.assertIs(self._floor_passed(encoder_replay=False), None) + self.assertIs( + self._floor_passed(encoder_replay=False, swa_replay_start=start), start + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/attention/unittests/dsv4/test_dsv41_bcg_hip.py b/test/registered/attention/unittests/dsv4/test_dsv41_bcg_hip.py new file mode 100644 index 0000000000..6049aed727 --- /dev/null +++ b/test/registered/attention/unittests/dsv4/test_dsv41_bcg_hip.py @@ -0,0 +1,277 @@ +"""HIP radix backend, low-ratio fork: breakable prefill graph replay and CP tail attention must match the eager build.""" + +import copy +import dataclasses +import unittest + +import torch + +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=25, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + + +def _extend_batch( + *, + seq_lens, + extend_lens, + req_pool_indices, + out_cache_loc, + device, +) -> ForwardBatch: + seq_lens_t = torch.tensor(seq_lens, dtype=torch.int32, device=device) + extend_t = torch.tensor(extend_lens, dtype=torch.int32, device=device) + positions = torch.cat( + [ + torch.arange(s - e, s, dtype=torch.int64, device=device) + for s, e in zip(seq_lens, extend_lens) + ] + ) + batch = ForwardBatch( + forward_mode=ForwardMode.EXTEND, + batch_size=len(seq_lens), + input_ids=torch.zeros(sum(extend_lens), dtype=torch.int64, device=device), + req_pool_indices=torch.tensor( + req_pool_indices, dtype=torch.int32, device=device + ), + seq_lens=seq_lens_t, + seq_lens_cpu=torch.tensor(seq_lens, dtype=torch.int32), + out_cache_loc=out_cache_loc, + seq_lens_sum=sum(seq_lens), + positions=positions, + ) + batch.extend_prefix_lens = seq_lens_t - extend_t + batch.extend_prefix_lens_cpu = [s - e for s, e in zip(seq_lens, extend_lens)] + batch.extend_seq_lens = extend_t + batch.extend_seq_lens_cpu = list(extend_lens) + batch.extend_start_loc = torch.cumsum(extend_t, dim=0) - extend_t + batch.extend_num_tokens = sum(extend_lens) + batch.global_num_token_non_padded_cpu = sum(extend_lens) + return batch + + +def _tensor_fields(obj): + for f in dataclasses.fields(obj): + value = getattr(obj, f.name, None) + if isinstance(value, torch.Tensor): + yield f.name, value + + +# torch.empty_like scratch: the index source writes every row before a consumer reads +# one, so two builds agree on shape, not contents +_UNINITIALIZED_SCRATCH_FIELDS = frozenset( + {"c1_sparse_raw_indices", "c2_sparse_raw_indices", "c4_sparse_raw_indices"} +) + + +@unittest.skipUnless( + is_hip() and torch.cuda.is_available(), "the HIP radix backend is ROCm only" +) +class TestHipBreakableGraphCaptureReplay(CustomTestCase): + """A captured store must read the refreshed target, and break-time metadata and + attention must equal the eager build for the same batch.""" + + BUCKET = 96 + + def test_capture_replay_matches_eager(self): + self._capture_replay_matches_eager(max_context_size=None) + + def _capture_replay_matches_eager(self, max_context_size): + from sglang.srt.model_executor.forward_context import ( + ForwardContext, + forward_context, + ) + from sglang.test.kits.attention_unittest.attention_methods.dsv4_attention import ( + DSV4_PAGE_SIZE, + DSV4AttentionCase, + _populate_swa_kv_cache, + build_dsv4_attention_fixture, + ) + + device = "cuda" + case = DSV4AttentionCase( + name="hip_bcg_extend", + backend="dsv4", + forward_mode=ForwardMode.EXTEND, + num_heads=64, + page_size=DSV4_PAGE_SIZE, + prefix_lens=(256, 256) if max_context_size is not None else (0, 0), + extend_lens=(40, 24), + ) + fixture = build_dsv4_attention_fixture( + self, + case, + device=device, + max_context_len=2 * DSV4_PAGE_SIZE, + compression_ratios=[0, 2, 1], + ) + self.addCleanup(fixture.runner._server_args_override.restore) + backend = fixture.backend + pool = fixture.runner.token_to_kv_pool + self.assertEqual(backend.low_ratios, (1, 2)) + _populate_swa_kv_cache( + fixture, max_context_len=2 * DSV4_PAGE_SIZE, device=device + ) + q_input, _ = fixture.actual_module.project(fixture.input_hidden) + live = fixture.forward_batch + live.global_num_token_non_padded_cpu = case.num_input_tokens + num_tokens = case.num_input_tokens + + def attention(forward_batch): + return backend.forward( + q=q_input, + k=q_input, + v=q_input, + layer=fixture.actual_module.attn, + forward_batch=forward_batch, + compress_ratio=0, + save_kv_cache=False, + attn_sink=fixture.actual_module.attn_sink, + ) + + # capture batch of a bucket: one request of BUCKET tokens on the static (all-zero) out_cache_loc slot + capture_batch = _extend_batch( + seq_lens=[self.BUCKET], + extend_lens=[self.BUCKET], + req_pool_indices=[0], + out_cache_loc=torch.zeros(self.BUCKET, dtype=torch.int64, device=device), + device=device, + ) + capture_batch.max_seq_len_override = max_context_size + # static view of the live batch: out_cache_loc padded to the bucket with the dummy slot + static = copy.copy(live) + static.max_seq_len_override = max_context_size + static.out_cache_loc = torch.nn.functional.pad( + live.out_cache_loc, (0, self.BUCKET - num_tokens), value=0 + ) + + with torch.no_grad(), forward_context(ForwardContext(attn_backend=backend)): + backend.init_forward_metadata(live) + eager = backend.forward_metadata + eager_out = attention(live) + if max_context_size is not None: + eager = backend._prefill_metadata_for_batch( + live, max_seq_len_override=max_context_size + ) + backend.forward_metadata = eager + backend.init_forward_metadata_in_graph(live) + + captured = backend.init_forward_metadata_for_breakable_cuda_graph_capture( + capture_batch + ) + self.assertIs(backend.forward_metadata, captured) + pinned = captured.core_attn_metadata.swa_out_cache_loc + self.assertEqual(tuple(pinned.shape), (self.BUCKET,)) + self.assertEqual( + captured.core_attn_metadata.page_table.shape[1], + ((max_context_size or self.BUCKET) + DSV4_PAGE_SIZE - 1) + // DSV4_PAGE_SIZE, + ) + + # A segment's store reads the target by address. + sink = torch.empty_like(pinned) + graph = torch.cuda.CUDAGraph() + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + sink.copy_(backend.get_swa_out_cache_loc(capture_batch)) + torch.cuda.current_stream().wait_stream(side) + with torch.cuda.graph(graph): + sink.copy_(backend.get_swa_out_cache_loc(capture_batch)) + + backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( + captured, live, static_forward_batch=static + ) + self.assertIs(backend.forward_metadata, captured) + self.assertIs(captured.core_attn_metadata.swa_out_cache_loc, pinned) + graph.replay() + torch.cuda.synchronize() + expected_target = pool.translate_loc_from_full_to_swa( + static.out_cache_loc + ).to(torch.int32) + self.assertTrue(torch.equal(sink, expected_target)) + self.assertTrue( + torch.equal( + pinned[:num_tokens], eager.core_attn_metadata.swa_out_cache_loc + ) + ) + + # Every break-time field is the eager build for the live batch. + for name, value in _tensor_fields(eager.core_attn_metadata): + if name == "swa_out_cache_loc": + continue + with self.subTest(field=name): + rebound = getattr(captured.core_attn_metadata, name) + if name in _UNINITIALIZED_SCRATCH_FIELDS: + self.assertEqual(rebound.shape, value.shape) + self.assertEqual(rebound.dtype, value.dtype) + self.assertEqual(rebound.device, value.device) + else: + self.assertTrue(torch.equal(rebound, value)) + for ratio in (1, 2): + for name in ("page_table", "compressed_seq_lens"): + with self.subTest(ratio=ratio, field=name): + self.assertTrue( + torch.equal( + getattr( + captured.low_ratio_indexer_metadata(ratio), name + ), + getattr(eager.low_ratio_indexer_metadata(ratio), name), + ) + ) + self.assertEqual(set(captured.fp4_low_ratio_prefill_workspaces), {1, 2}) + + # The attention break on the refreshed metadata is the eager attention. + replay_out = attention(live) + self.assertTrue(torch.equal(replay_out, eager_out)) + + # A second replay with a different batch shape rebinds again. + live2 = _extend_batch( + seq_lens=[24, 40], + extend_lens=[24, 40], + req_pool_indices=[1, 0], + out_cache_loc=torch.flip(live.out_cache_loc, dims=[0]), + device=device, + ) + static2 = copy.copy(live2) + static2.max_seq_len_override = max_context_size + static2.out_cache_loc = torch.nn.functional.pad( + live2.out_cache_loc, (0, self.BUCKET - num_tokens), value=0 + ) + backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( + captured, live2, static_forward_batch=static2 + ) + graph.replay() + torch.cuda.synchronize() + self.assertTrue( + torch.equal( + sink, + pool.translate_loc_from_full_to_swa(static2.out_cache_loc).to( + torch.int32 + ), + ) + ) + backend.init_forward_metadata(live2) + eager2 = backend.forward_metadata + self.assertTrue( + torch.equal( + captured.core_attn_metadata.seq_lens_casual, + eager2.core_attn_metadata.seq_lens_casual, + ) + ) + if max_context_size is not None: + overflow = copy.copy(live2) + overflow.seq_lens_cpu = torch.tensor([max_context_size + 1, 40]) + with self.assertRaisesRegex( + ValueError, "smaller than the live context" + ): + backend.prepare_forward_metadata_for_breakable_cuda_graph_replay( + captured, overflow, static_forward_batch=static2 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py b/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py new file mode 100644 index 0000000000..a930ed6729 --- /dev/null +++ b/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py @@ -0,0 +1,397 @@ +"""The fused low-ratio compressor must write the real cache pool byte-identical to the unfused chain.""" + +import types +import unittest + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend +from sglang.srt.runtime_context import get_context +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=25, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + +HIDDEN = 2048 +HEAD_DIM = 512 +# Read ratio-1/2 source layers from the pool rather than assuming the model ordering. +RATIOS = [0] * 2 + [2] * 18 + [1] * 20 +KV_SOURCE_LAYERS = (2, 8, 14, 20) +POOL_PAGE_SIZE = 256 +POOL_FULL_SIZE = 4 * POOL_PAGE_SIZE +MAX_REQS = 256 +INDEX_HEAD_DIM = 128 +ROPE_DIM = 64 +EPS = 1e-6 + + +def _make_pool(): + """The real DeepSeekV4TokenToKVPool, so the buffers, their dtypes, the page + sizes and the pair-state ring all come from the code under test.""" + from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool + + return DeepSeekV4TokenToKVPool( + max_num_reqs=MAX_REQS, + swa_size=POOL_FULL_SIZE, + c4_size=0, + c128_size=0, + c4_state_pool_size=0, + c128_state_pool_size=0, + page_size=POOL_PAGE_SIZE, + swa_page_size=128, + dtype=torch.float8_e4m3fn, + c4_state_dtype=torch.float32, + c128_state_dtype=torch.float32, + qk_nope_head_dim=HEAD_DIM - ROPE_DIM, + qk_rope_head_dim=ROPE_DIM, + indexer_head_dim=INDEX_HEAD_DIM, + layer_num=len(RATIOS), + device="cuda", + enable_memory_saver=False, + compression_ratios=RATIOS, + kv_source_layers=KV_SOURCE_LAYERS, + full_size=POOL_FULL_SIZE, + ) + + +def _build(n: int, ratio: int, seed: int): + """One decode step against the real pool.""" + from sglang.srt.layers.attention.deepseek_v4_backend import ( + _low_ratio_compression_metadata, + ) + from sglang.srt.layers.attention.dsv4.dsv41_sparse import ( + DeepseekV41Compressor, + DeepseekV41Indexer, + RMSNorm, + ) + from sglang.srt.model_loader.utils import set_default_torch_dtype + + g = torch.Generator(device="cuda").manual_seed(seed) + with set_default_torch_dtype(torch.bfloat16): + compressor = DeepseekV41Compressor( + HIDDEN, HEAD_DIM, ratio, EPS, fused_compress=True + ).cuda() + k_norm = RMSNorm(INDEX_HEAD_DIM, EPS).cuda() + wk = torch.nn.Linear( + HEAD_DIM, INDEX_HEAD_DIM, bias=False, dtype=torch.bfloat16 + ).cuda() + # Not ones anywhere: a constant weight cannot catch a wrong element index. + with torch.no_grad(): + for module in (compressor, k_norm, wk): + for param in module.parameters(): + param.copy_(torch.randn(param.shape, generator=g, device="cuda") * 0.1) + x = torch.randn(n, HIDDEN, generator=g, device="cuda", dtype=torch.bfloat16) + + # Every token is the last of its group, so every row completes and writes. + pos = torch.arange(n, device="cuda", dtype=torch.int64) * ratio + (ratio - 1) + raw_out_loc = (torch.arange(n, device="cuda", dtype=torch.int32) + 1) * ratio + ( + ratio - 1 + ) + out_loc, _ = _low_ratio_compression_metadata( + ratio, raw_out_loc.to(torch.int64) + 1, raw_out_loc + ) + + ang = torch.randn(int(pos.max()) + 2, ROPE_DIM // 2, generator=g, device="cuda") + freqs = torch.polar(torch.ones_like(ang), ang) + + pool = _make_pool() + layer_id = pool.sources_by_ratio[ratio][0] + # Use the real pool: get_extra_key_buffer exposes a uint8 store as an fp8 view. + kv_cache = pool.get_extra_key_buffer(layer_id) + compressed_page = pool.get_extra_key_page_size(layer_id) + # ROCm keeps payload and scale split; CUDA fuses them into one 68-byte row + split = pool.low_ratio_index_k_is_split(layer_id) + if split: + index_cache = ( + pool.get_index_k_fp4_payload_buffer(layer_id).view(torch.uint8), + pool.get_index_k_fp4_scale_buffer(layer_id), + ) + else: + index_cache = (pool.get_index_k_with_scale_buffer(layer_id),) + # Ratio 1 pools nothing, so it has no pair state -- the pool asserts on the + # accessor rather than returning an empty one. + state = pool.get_attention_compress_states(layer_id) if ratio == 2 else None + assert int(out_loc.max()) < kv_cache.shape[0] * compressed_page, ( + "the test's slots must fit the pool" + ) + + indexer = types.SimpleNamespace( + owns_k=True, + wk=wk, + k_norm=k_norm, + rope_head_dim=ROPE_DIM, + index_head_dim=INDEX_HEAD_DIM, + ) + # The real method, bound to the stand-in: it reads only wk and + # index_head_dim, and it is what decides whether the in-tree GEMM serves wk. + captured = {} + + def forward_wk(latent): + projected = DeepseekV41Indexer.forward_wk(indexer, latent) + captured.update(latent=latent, projected=projected) + return projected + + indexer.forward_wk = forward_wk + + if state is not None: + # Seeded rather than zeroed, so a completing row that fails to read its + # partner shows up instead of pairing against zeros. + state.kv_score_buffer.kv_score.copy_( + torch.randn( + state.kv_score_buffer.kv_score.shape, + generator=g, + device="cuda", + dtype=torch.float32, + ) + ) + layer = types.SimpleNamespace( + compressor=compressor, + indexer=indexer, + layer_id=layer_id, + compress_ratio=ratio, + rope_head_dim=ROPE_DIM, + freqs_cis=freqs, + ) + backend = types.SimpleNamespace( + token_to_kv_pool=pool, + forward_metadata=types.SimpleNamespace( + core_metadata=types.SimpleNamespace( + raw_out_loc=raw_out_loc, c1_out_loc=out_loc, c2_out_loc=out_loc + ) + ), + ) + return types.SimpleNamespace( + backend=backend, + layer=layer, + x=x, + pos=pos, + freqs=freqs, + out_loc=out_loc, + kv_cache=kv_cache, + index_cache=index_cache, + index_k_split=split, + compressor=compressor, + k_norm=k_norm, + captured=captured, + pair_state=( + state.kv_score_buffer.kv_score.clone() if state is not None else None + ), + ring_size=state.ring_size if state is not None else 0, + compressed_page=compressed_page, + req=torch.arange(n, device="cuda", dtype=torch.int64), + ) + + +def _reference(t, ratio: int): + """Check numerics independently, then check byte-exact packing of actual latents.""" + from sglang.kernels.ops.attention.dsv4.attn import fused_store_cache + from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + index_k_rope_pack, + ) + from sglang.kernels.ops.attention.dsv4.torch_quant import fake_quant_compressed_kv + from sglang.srt.layers.attention.dsv4.dsv41_sparse import rope_tail + + if ratio == 1: + pooled = t.compressor.wkv(t.x) + else: + fused = t.compressor.project_fused(t.x) + kv, score = fused[..., :HEAD_DIM], fused[..., HEAD_DIM:] + # translate_from_req_position_to_state_loc for the slot pos - 1 + # left: the ring puts the read and the write on different rows. + read = t.req * t.ring_size + (t.pos - 1) % t.ring_size + partner_kv = t.pair_state[read, :HEAD_DIM] + partner_score = t.pair_state[read, HEAD_DIM:] + if hasattr(t, "draft_len"): + # In-block predecessors come from this projection, not stale ring + # slots. This reference deliberately uses the unfused torch chain. + in_block = torch.arange(t.pos.numel(), device="cuda") % t.draft_len != 0 + partner_kv = torch.where(in_block[:, None], kv.roll(1, 0), partner_kv) + partner_score = torch.where( + in_block[:, None], score.roll(1, 0), partner_score + ) + pooled = ( + torch.stack([partner_kv, kv], dim=1) + * torch.stack([partner_score, score], dim=1).softmax(dim=1) + ).sum(dim=1) + live = t.out_loc > 0 + latent = t.captured["latent"] + projected = t.captured["projected"] + if live.any(): + normalized_input = pooled[live].bfloat16().double() + normalized = normalized_input * torch.rsqrt( + normalized_input.square().mean(dim=-1, keepdim=True) + EPS + ) + expected_latent = normalized * t.compressor.norm.weight.double() + # Pooling and normalization may differ by up to two bf16 ULPs. + torch.testing.assert_close( + latent[live].float(), expected_latent.float(), rtol=2**-6, atol=2**-20 + ) + expected_projection = F.linear( + latent[live].float(), t.layer.indexer.wk.weight.float() + ) + torch.testing.assert_close( + projected[live].float(), expected_projection, rtol=1e-2, atol=1e-2 + ) + + group_pos = t.pos & ~(ratio - 1) + # fused_store_cache takes uint8, while the pool exposes an fp8 view; + # compare and call any() through uint8 views of both buffers. + kv_cache = torch.zeros( + t.kv_cache.shape, dtype=torch.uint8, device=t.kv_cache.device + ) + if not live.any(): + return kv_cache, tuple(torch.zeros_like(buf) for buf in t.index_cache) + fused_store_cache( + input=fake_quant_compressed_kv( + rope_tail(latent[live], t.freqs[group_pos[live]], ROPE_DIM) + ), + cache=kv_cache, + indices=t.out_loc[live], + page_size=t.compressed_page, + type="flashmla", + ) + index_cache = tuple(torch.zeros_like(buf) for buf in t.index_cache) + if t.index_k_split: + from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + store_fp4_index_k_cache_split, + ) + from sglang.srt.layers.attention.dsv4.dsv41_sparse import _rope_fq4 + + # the unfused ROCm chain: index_keys then the split-layout writer + payload, scale = index_cache + store_fp4_index_k_cache_split( + _rope_fq4(t.k_norm(projected[live]), t.freqs[group_pos[live]], ROPE_DIM), + payload.view(torch.float4_e2m1fn_x2), + scale, + t.out_loc[live], + page_size=payload.shape[3], + rne=True, + ) + else: + index_k_rope_pack( + t.k_norm(projected[live]), + t.freqs[group_pos[live]], + ROPE_DIM, + cache=index_cache[0], + loc=t.out_loc[live], + ) + return kv_cache, index_cache + + +class TestFusedLowRatioCompress(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + # The pool reads the scheduler's config bags (spec, for the pair-ring + # size), so they have to be published before one can be built. + override = get_context().override_server_args(page_size=POOL_PAGE_SIZE) + override.install() + cls.addClassCleanup(override.restore) + + def _check_step(self, t, ratio: int): + DeepseekV4AttnBackend._low_ratio_compress_fused( + t.backend, t.layer, t.x, t.req, t.pos + ) + ref_kv, ref_index = _reference(t, ratio) + torch.cuda.synchronize() + + got_kv = t.kv_cache.view(torch.uint8) + if (t.out_loc > 0).any(): + self.assertTrue(got_kv.any(), "the main-KV write published nothing") + for got in t.index_cache: + self.assertTrue(got.any(), "the index-K write published nothing") + self.assertTrue( + torch.equal(got_kv, ref_kv), + f"{ratio=}: {int((got_kv != ref_kv).sum())} main-KV cache " + f"bytes differ from the unfused chain", + ) + for got, ref in zip(t.index_cache, ref_index): + self.assertTrue( + torch.equal(got, ref), + f"{ratio=}: {int((got != ref).sum())} index-K " + f"cache bytes differ from the unfused chain", + ) + + def test_matches_the_unfused_chain(self): + for ratio in (1, 2): + with self.subTest(ratio=ratio): + self._check_step(_build(64, ratio, seed=164 + ratio), ratio) + + @unittest.skipUnless(is_hip(), "HIP fused pair-state writer") + def test_pair_state_survives_the_next_decode_step(self): + t = _build(8, 2, seed=3000) + t.req.copy_(t.req.roll(1)) + state = t.backend.token_to_kv_pool.get_attention_compress_states( + t.layer.layer_id + ).kv_score_buffer.kv_score + complete_slots = t.out_loc.clone() + t.pos.sub_(1) + t.out_loc.fill_(-1) + projected = t.compressor.project_fused(t.x) + expected_state = state.clone() + write_rows = t.req * t.ring_size + t.pos % t.ring_size + expected_state[write_rows] = projected + self._check_step(t, 2) + self.assertTrue(torch.equal(state, expected_state)) + + t.pair_state = expected_state + t.pos.add_(1) + t.out_loc.copy_(complete_slots) + t.x.normal_() + self._check_step(t, 2) + self.assertTrue(torch.equal(state, expected_state)) + + @unittest.skipUnless(is_hip(), "HIP fused compressor regression") + def test_padded_rows_publish_nothing(self): + """A padded graph suffix carries raw_out_loc == 0 and out_loc == 0, which + both kernels must read off the arrays (the caller passes no mask).""" + n, pad = 8, 3 + for ratio in (1, 2): + with self.subTest(ratio=ratio): + t = _build(n, ratio, seed=200 + ratio) + core = t.backend.forward_metadata.core_metadata + core.raw_out_loc[-pad:] = 0 + core.c1_out_loc[-pad:] = 0 + core.c2_out_loc[-pad:] = 0 + DeepseekV4AttnBackend._low_ratio_compress_fused( + t.backend, t.layer, t.x, t.req, t.pos + ) + torch.cuda.synchronize() + # slot 0 of page 0 is the reserved dummy a padded row lands on without the predicate + self.assertFalse( + t.kv_cache.view(torch.uint8)[0, :576].any(), + "a padded row wrote main-KV slot 0", + ) + if t.index_k_split: + payload, scale = t.index_cache + self.assertFalse(payload[0, 0, :, 0, :].any()) + self.assertFalse(scale[0, 0, :, 0].any()) + else: + self.assertFalse(t.index_cache[0][0, : INDEX_HEAD_DIM // 2].any()) + + @unittest.skipUnless(is_hip(), "HIP fused compressor regression") + def test_open_group_rows_publish_nothing(self): + """A live ratio-2 token at an even position completes no group, so the metadata + gives it c2_out_loc == -1; an index-K writer taking that -1 straight from + the array would wrap to the last slot.""" + from sglang.srt.layers.attention.deepseek_v4_backend import ( + _low_ratio_compression_metadata, + ) + + n, pending, ratio = 8, 3, 2 + t = _build(n, ratio, seed=300) + core = t.backend.forward_metadata.core_metadata + # One position back: even, so the tail's groups are still open. + t.pos[-pending:] -= 1 + out_loc, _ = _low_ratio_compression_metadata(ratio, t.pos + 1, core.raw_out_loc) + self.assertTrue(bool((out_loc[-pending:] == -1).all()), out_loc.tolist()) + self.assertTrue(bool((out_loc[:-pending] > 0).all()), out_loc.tolist()) + # _build hands the same tensor out as c2_out_loc and t.out_loc. + core.c2_out_loc.copy_(out_loc) + self._check_step(t, ratio) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py b/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py new file mode 100644 index 0000000000..c1cd9d2858 --- /dev/null +++ b/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py @@ -0,0 +1,493 @@ +"""The aiter_sparse ROCm attention backend and its split-KV combine must match the torch reference and aiter's own reduce on the served packed fp8 KV layout, bitwise repeatable and batch-invariant.""" + +import math +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=25, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") + + +NOPE, ROPE, D = 448, 64, 512 +PAGE = 256 +BYTES = 584 +SCALE = D**-0.5 + + +# Relative tolerance for the short-list case: about 3x the measured aiter error +# (~3e-3) and 40x under what attending one stray key on a 5-key list costs (~0.4). +TOL_SHORT = 1e-2 + + +def _pack_cache(num_blocks, device, gen, *, fp8_view=True): + """Random bf16 keys in the packed fp8 layout: cache [num_blocks, PAGE, 1, BYTES] and the dequantized keys [slots, D] fp32.""" + slots = num_blocks * PAGE + k = torch.randn(slots, D, generator=gen) * 0.5 + nope = k[:, :NOPE].reshape(slots, NOPE // 64, 64) + amax = nope.abs().amax(-1, keepdim=True).clamp(min=1e-6) + exp = torch.ceil(torch.log2(amax / 448.0)).clamp(min=-127, max=127) + scale = torch.pow(2.0, exp) + nope_fp8 = (nope / scale).to(torch.float8_e4m3fn) + nope_deq = nope_fp8.float() * scale + rope = k[:, NOPE:].to(torch.bfloat16) + raw = torch.zeros(num_blocks, PAGE * BYTES, dtype=torch.uint8) + data = raw[:, : PAGE * 576].view(num_blocks, PAGE, 576) + data[:, :, :NOPE] = nope_fp8.view(torch.uint8).reshape(num_blocks, PAGE, NOPE) + data[:, :, NOPE:] = rope.view(torch.uint8).reshape(num_blocks, PAGE, 2 * ROPE) + scales = raw[:, PAGE * 576 :].view(num_blocks, PAGE, 8) + scales[:, :, :7] = (exp.reshape(num_blocks, PAGE, 7) + 127).to(torch.uint8) + cache = raw.view(num_blocks, PAGE, 1, BYTES) + if fp8_view: + cache = cache.view(torch.float8_e4m3fn) + deq = torch.cat([nope_deq.reshape(slots, NOPE), rope.float()], dim=1).to(device) + return cache.to(device), deq + + +def _masked(indices, lengths): + """One length-folded list; _fold_lengths_into_index_lists returns the (main, extra) pair.""" + from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( + _fold_lengths_into_index_lists, + ) + + return _fold_lengths_into_index_lists(indices, lengths)[0] + + +def _decode_case(batch, heads, gen, dev): + """Two SWA and five top-k pages of packed keys, a query per row, the sink, and one + random 128-slot SWA list plus one 512-slot top-k list per row ([b, 1, w] int32).""" + swa_cache, swa_deq = _pack_cache(2, dev, gen) + topk_cache, topk_deq = _pack_cache(5, dev, gen) + q = (torch.randn(batch, 1, heads, D, generator=gen) * 0.5).to(torch.bfloat16) + sink = (torch.randn(heads, generator=gen) * 0.5).to(dev) + swa_idx = torch.stack( + [torch.randperm(2 * PAGE, generator=gen)[:128] for _ in range(batch)] + ) + topk_idx = torch.stack( + [torch.randperm(5 * PAGE, generator=gen)[:512] for _ in range(batch)] + ) + return SimpleNamespace( + swa_cache=swa_cache, + swa_deq=swa_deq, + topk_cache=topk_cache, + topk_deq=topk_deq, + q=q.to(dev), + sink=sink, + swa_idx=swa_idx.to(torch.int32).unsqueeze(1).to(dev), + topk_idx=topk_idx.to(torch.int32).unsqueeze(1).to(dev), + ) + + +def _reference(q, sink, sets): + """Decode form of _reference_prefill: sets = [(deq_keys, indices [b, 1, w], + lengths [b])], a slot at or past its row's length being padding.""" + folded = [] + for deq, idx, length in sets: + pos = torch.arange(idx.shape[-1], device=idx.device) + folded.append((deq, torch.where(pos < length[:, None, None], idx, -1))) + return _reference_prefill(q, sink, folded) + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported(), "aiter gluon kernel is gfx950-only" +) +class TestAiterSparseBackend(CustomTestCase): + def _assert_matches_reference( + self, + batch, + heads, + swa_lengths, + topk_lengths, + seed=0, + tol=3e-2, + padding_key_inside_length=False, + ): + from sglang.srt.layers.attention.hip_flash_mla import ( + flash_mla_with_kvcache_entrypoint, + ) + + gen = torch.Generator(device="cpu").manual_seed(seed) + dev = torch.device("cuda") + c = _decode_case(batch, heads, gen, dev) + q, sink, swa_idx, topk_idx = c.q, c.sink, c.swa_idx, c.topk_idx + swa_len = torch.tensor(swa_lengths, dtype=torch.int32, device=dev) + topk_len = torch.tensor(topk_lengths, dtype=torch.int32, device=dev) + if padding_key_inside_length: + topk_idx[:, 0, 3] = -1 + ref = _reference( + q, sink, [(c.swa_deq, swa_idx, swa_len), (c.topk_deq, topk_idx, topk_len)] + ) + # The backend folds the lengths into the index lists before this call. + kwargs = dict( + backend="aiter_sparse", + q=q, + k_cache=c.swa_cache, + head_dim_v=D, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=None, + softmax_scale=SCALE, + is_fp8_kvcache=True, + attn_sink=sink, + extra_k_cache=c.topk_cache, + indices=_masked(swa_idx, swa_len), + topk_length=swa_len, + extra_indices_in_kvcache=_masked(topk_idx, topk_len), + extra_topk_length=topk_len, + ) + got = flash_mla_with_kvcache_entrypoint(**kwargs)[0] + self.assertEqual(got.shape, q.shape) + self.assertEqual(got.dtype, torch.bfloat16) + err = (got.float() - ref).abs().max().item() / ref.abs().max().item() + self.assertLess(err, tol, f"aiter vs reference {err:.4f}") + for _ in range(3): + self.assertTrue( + torch.equal(flash_mla_with_kvcache_entrypoint(**kwargs)[0], got) + ) + + def test_folded_inverse_rope_matches_the_model_rope(self): + """The split combine folds the model's inverse RoPE into its output: bitwise the + combine without it followed by the standalone inverse RoPE of the rope tail.""" + from sglang.srt.layers.attention.hip_flash_mla import ( + flash_mla_with_kvcache_entrypoint, + ) + + gen = torch.Generator(device="cpu").manual_seed(20) + dev = torch.device("cuda") + fr = _freqs(dev) + for batch, pos_dtype in ((1, torch.int64), (5, torch.int32)): + with self.subTest(batch=batch): + c = _decode_case(batch, 16, gen, dev) + swa_len = torch.full((batch,), 77, dtype=torch.int32, device=dev) + topk_len = torch.full((batch,), 301, dtype=torch.int32, device=dev) + kwargs = dict( + backend="aiter_sparse", + q=c.q, + k_cache=c.swa_cache, + head_dim_v=D, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=None, + softmax_scale=SCALE, + is_fp8_kvcache=True, + attn_sink=c.sink, + extra_k_cache=c.topk_cache, + indices=_masked(c.swa_idx, swa_len), + topk_length=swa_len, + extra_indices_in_kvcache=_masked(c.topk_idx, topk_len), + extra_topk_length=topk_len, + ) + pos = torch.randint(0, 8192, (batch,), device=dev, dtype=pos_dtype) + # the combine runs only with split KV + with patch( + "sglang.srt.layers.attention.hip_flash_mla.hip_attn_kv_splits", + lambda: 4, + ): + plain = flash_mla_with_kvcache_entrypoint(**kwargs)[0] + got = flash_mla_with_kvcache_entrypoint( + **kwargs, inv_rope=(fr, pos) + )[0] + ref = plain.clone().squeeze(1) + _model_inverse_rope(ref[..., -ROPE:], fr, pos) + self.assertTrue(torch.equal(got.squeeze(1), ref)) + + def test_matches_reference(self): + """Full lists, contexts shorter than the window and the top-k width (the length + masks live slots left in the list), and the model's 64-padded heads; then + 5-key lists with a -1 inside the length (index 3 of the top-k list), which + must not be attended: on a 5-key list a stray key moves the softmax mass past + the tolerance, where the 640-key case absorbs it.""" + cases = ( + (3, 16, [101, 128, 5], [100, 512, 1], 1, 3e-2, False), + (2, 16, [2, 3], [4, 5], 6, TOL_SHORT, True), + ) + for batch, heads, swa_lengths, topk_lengths, seed, tol, pad in cases: + with self.subTest(batch=batch, swa=swa_lengths, topk=topk_lengths): + self._assert_matches_reference( + batch, + heads, + swa_lengths, + topk_lengths, + seed=seed, + tol=tol, + padding_key_inside_length=pad, + ) + + ROWS = 96 # aiter picks 2 splits here, 4 at 1..64 rows + + def _run_rows(self, batch): + """Row i of every batch is row i of the same 96-row case.""" + from sglang.srt.layers.attention.hip_flash_mla import ( + flash_mla_with_kvcache_entrypoint, + ) + + dev = torch.device("cuda") + c = _decode_case( + self.ROWS, 16, torch.Generator(device="cpu").manual_seed(40), dev + ) + lens = lambda n: torch.full((batch,), n, dtype=torch.int32, device=dev) + return flash_mla_with_kvcache_entrypoint( + backend="aiter_sparse", + q=c.q[:batch], + k_cache=c.swa_cache, + head_dim_v=D, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=None, + softmax_scale=SCALE, + is_fp8_kvcache=True, + attn_sink=c.sink, + extra_k_cache=c.topk_cache, + indices=_masked(c.swa_idx[:batch], lens(128)), + extra_indices_in_kvcache=_masked(c.topk_idx[:batch], lens(512)), + )[0] + + def test_pinned_splits_are_batch_invariant(self): + """A pinned split-KV count (deterministic inference pins 4) keeps a row's output + bitwise the same at every batch size; aiter's cost model changes the count + past 64 rows.""" + with patch( + "sglang.srt.layers.attention.hip_flash_mla.hip_attn_kv_splits", lambda: 4 + ): + one, eight, many = ( + self._run_rows(1), + self._run_rows(8), + self._run_rows(self.ROWS), + ) + self.assertTrue(torch.equal(one, eight[:1])) + self.assertTrue(torch.equal(eight, many[:8])) + + +SWA, TOPK = 128, 512 + + +def _prefill_lists(num_tokens, device, gen): + """Causal prefill lists as the indexer emits them: SWA holds the 128 most recent + slots, top-k 512 random earlier slots, both -1 padded.""" + pos = torch.arange(num_tokens) + swa = pos[:, None] - torch.arange(SWA)[None, :] + swa = torch.where(swa >= 0, swa, torch.full_like(swa, -1)) + topk = torch.argsort(torch.rand(num_tokens, num_tokens, generator=gen), dim=1) + topk = topk[:, :TOPK] + topk = torch.where(topk <= pos[:, None], topk, torch.full_like(topk, -1)) + topk = topk.sort(dim=1, descending=True).values + return ( + swa.to(torch.int32).view(num_tokens, 1, SWA).to(device), + topk.to(torch.int32).view(num_tokens, 1, TOPK).to(device), + ) + + +def _reference_prefill(q, sink, sets, chunk=256): + """q [t, 1, h, D] bf16; sets = [(deq_keys [slots, D], indices [t, 1, w])].""" + t, _, h, _ = q.shape + out = torch.empty(t, 1, h, D, device=q.device, dtype=torch.float32) + for t0 in range(0, t, chunk): + t1 = min(t, t0 + chunk) + idx = torch.cat([i[t0:t1, 0] for _, i in sets], dim=1) + keys = torch.cat( + [deq[i[t0:t1, 0].clamp(min=0).long()] for deq, i in sets], dim=1 + ) + s = torch.einsum("chd,cnd->chn", q[t0:t1, 0].float(), keys) * SCALE + s = torch.where((idx >= 0)[:, None, :], s, torch.full_like(s, float("-inf"))) + logits = torch.cat([s, sink[None, :, None].expand(t1 - t0, h, 1)], dim=-1) + p = torch.softmax(logits, dim=-1)[..., :-1] + out[t0:t1, 0] = torch.einsum("chn,cnd->chd", p, keys) + return out + + +@unittest.skipUnless( + is_hip() and is_gfx95_supported(), "aiter gluon kernel is gfx950-only" +) +class TestAiterSparsePrefill(CustomTestCase): + NUM_TOKENS = 2048 + HEADS = 16 + + @classmethod + def setUpClass(cls): + gen = torch.Generator(device="cpu").manual_seed(0) + dev = torch.device("cuda") + blocks = cls.NUM_TOKENS // PAGE + 1 + cls.swa_cache, swa_deq = _pack_cache(blocks, dev, gen, fp8_view=False) + cls.topk_cache, topk_deq = _pack_cache(blocks, dev, gen, fp8_view=False) + cls.q = ( + (torch.randn(cls.NUM_TOKENS, 1, cls.HEADS, D, generator=gen) * 0.5) + .to(torch.bfloat16) + .to(dev) + ) + cls.sink = (torch.randn(cls.HEADS, generator=gen) * 0.5).to(dev) + cls.swa_idx, cls.topk_idx = _prefill_lists(cls.NUM_TOKENS, dev, gen) + cls.ref = _reference_prefill( + cls.q, cls.sink, [(swa_deq, cls.swa_idx), (topk_deq, cls.topk_idx)] + ) + + def _run(self, backend, rows=slice(None)): + from sglang.srt.layers.attention.hip_flash_mla import ( + flash_mla_with_kvcache_entrypoint, + ) + + return flash_mla_with_kvcache_entrypoint( + backend=backend, + q=self.q[rows], + k_cache=self.swa_cache, + head_dim_v=D, + block_table=None, + cache_seqlens=None, + tile_scheduler_metadata=None, + softmax_scale=SCALE, + is_fp8_kvcache=True, + indices=self.swa_idx[rows], + topk_length=None, + attn_sink=self.sink, + extra_k_cache=self.topk_cache, + extra_indices_in_kvcache=self.topk_idx[rows], + extra_topk_length=None, + )[0] + + def test_matches_reference(self): + got = self._run("aiter_sparse") + self.assertEqual(got.shape, self.q.shape) + self.assertEqual(got.dtype, torch.bfloat16) + err = (got.float() - self.ref).abs().max().item() / self.ref.abs().max().item() + # Measured 2.5e-3 (bf16 q, bf16 probabilities). + self.assertLess(err, 1e-2, f"aiter vs reference {err:.2e}") + for _ in range(3): + self.assertTrue(torch.equal(self._run("aiter_sparse"), got)) + + def test_batch_invariant(self): + from sglang.srt.layers.attention import hip_flash_mla + + # both batches are at or above the unsplit threshold, so each row runs one program in the same order + half = self.NUM_TOKENS // 2 + self.assertGreaterEqual( + half, hip_flash_mla._AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS + ) + full = self._run("aiter_sparse") + part = self._run("aiter_sparse", rows=slice(0, half)) + self.assertTrue(torch.equal(full[:half], part)) + + +def _freqs(device, max_pos=8192, seed=0): + gen = torch.Generator(device="cpu").manual_seed(seed) + angles = torch.rand(max_pos, ROPE // 2, generator=gen) * 2 * math.pi + freqs_cis = torch.polar(torch.ones_like(angles), angles).to(device) + return torch.view_as_real(freqs_cis).flatten(-2).contiguous() + + +def _model_inverse_rope(x, freqs_real, positions): + """The model's standalone inverse RoPE of the attention output: + fused_rope_inplace(..., inverse=True) with the batched flat kernel + (set_batched_rope(True)).""" + from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope + from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace + + set_batched_rope(True) + freqs_cis = torch.view_as_complex(freqs_real.view(freqs_real.shape[0], -1, 2)) + fused_rope_inplace(x, None, freqs_cis, positions, inverse=True) + + +@unittest.skipUnless(is_hip(), "HIP radix backend") +class TestDecodeSelectionOrder(CustomTestCase): + """The HIP decode top-k must be ordered by position, not slot: the aiter sparse + kernel sums in list order, so a slot-ordered row makes the attention bits depend + on which pages a request landed on.""" + + def test_position_ordered_selection_is_page_invariant(self): + """Same keys on two page layouts: the position-sorted selection attends bitwise + the same, and the AOT sort with raw indices produces that order.""" + from sglang.kernels.ops.attention.dsv4.attn import fused_store_cache + from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( + topk_transform_paged_sorted, + ) + from sglang.srt.layers.attention.hip_flash_mla import aiter_sparse_decode_fwd + + torch.manual_seed(0) + dev = "cuda" + k_a = torch.randn(PAGE, D, device=dev, dtype=torch.bfloat16) + k_b = torch.randn(PAGE, D, device=dev, dtype=torch.bfloat16) + k_t = torch.randn(1, D, device=dev, dtype=torch.bfloat16) + q = torch.randn(1, 1, 16, D, device=dev, dtype=torch.bfloat16) + sink = torch.zeros(16, device=dev, dtype=torch.float32) + no_swa = torch.full((1, 1, 128), -1, device=dev, dtype=torch.int32) + # 513 positions, drop position 356 (in the middle of the second page) + scores = torch.zeros(1, 1024, device=dev, dtype=torch.float32) + scores[0, 356] = -1.0 + seq_lens = torch.tensor([513], device=dev, dtype=torch.int32) + + outs = [] + for page_b, page_t in ((3, 4), (4, 3)): + cache = torch.zeros(8, PAGE * BYTES, dtype=torch.uint8, device=dev) + slots = { + "a": torch.arange(PAGE, 2 * PAGE, device=dev), + "b": torch.arange(page_b * PAGE, page_b * PAGE + PAGE, device=dev), + "t": torch.tensor([page_t * PAGE], device=dev), + } + for name, k in (("a", k_a), ("b", k_b), ("t", k_t)): + fused_store_cache( + k, cache, slots[name], page_size=PAGE, type="flashmla" + ) + page_table = torch.tensor( + [[1, page_b, page_t, 0]], device=dev, dtype=torch.int32 + ) + page_indices = torch.full((1, 512), -1, device=dev, dtype=torch.int32) + raw_indices = torch.full((1, 512), -1, device=dev, dtype=torch.int32) + topk_transform_paged_sorted( + scores, seq_lens, page_table, page_indices, PAGE, raw_indices + ) + expected_positions = torch.cat( + [torch.arange(0, 356, device=dev), torch.arange(357, 513, device=dev)] + ).to(torch.int32) + torch.testing.assert_close(raw_indices[0], expected_positions) + out, _ = aiter_sparse_decode_fwd( + q=q, + k_cache=cache.view(8, PAGE, 1, BYTES), + indices=no_swa, + attn_sink=sink, + softmax_scale=D**-0.5, + extra_k_cache=cache.view(8, PAGE, 1, BYTES), + extra_indices_in_kvcache=page_indices.view(1, 1, 512), + ) + outs.append(out.clone()) + self.assertTrue( + torch.equal(outs[0], outs[1]), "attention bits follow the page layout" + ) + + +@unittest.skipIf(not is_hip(), "HIP only") +class TestUniformIndptrGraphs(CustomTestCase): + def test_graph_does_not_read_row_pointers_another_graph_records(self): + """Two graphs of one shape (the V4.1 verify variants): the one replayed alone must + not read row pointers only the other graph writes.""" + from sglang.srt.layers.attention import hip_flash_mla + + hip_flash_mla._UNIFORM_INDPTR_CACHE.clear() + n, width, dev = 18, 512, "cuda" + expected = torch.arange( + 0, (n + 1) * width, width, dtype=torch.int32, device=dev + ) + outs, seen = [], [] + graphs = [torch.cuda.CUDAGraph() for _ in range(2)] + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + for g in graphs: + out = torch.zeros_like(expected) + with torch.cuda.graph(g, stream=stream): + indptr = hip_flash_mla._uniform_indptr(n, width, dev) + out.copy_(indptr) + outs.append(out) + seen.append(indptr) + torch.cuda.synchronize() + # the first graph never replays: poison whatever row pointers it was handed + seen[0].fill_(-7) + graphs[1].replay() + torch.cuda.synchronize() + torch.testing.assert_close(outs[1], expected, rtol=0, atol=0) + self.assertNotIn((n, width, dev), hip_flash_mla._UNIFORM_INDPTR_CACHE) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py b/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py index 612e5e07fb..f68f4f036d 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_aiter_hip.py @@ -3,6 +3,7 @@ import sys import types import unittest +from types import SimpleNamespace from unittest.mock import patch import torch @@ -209,5 +210,55 @@ class TestAiterMHCGLM53Flash(CustomTestCase): ) +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "requires gfx950") +class TestMhcPostSplitH(CustomTestCase): + """The model's hc_post must match aiter's mhc_post bitwise, eagerly and under graph replay.""" + + def setUp(self): + from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + + self.layer = SimpleNamespace( + config=SimpleNamespace(model_type="deepseek_v41"), hc_mult=4 + ) + self.run_post = lambda *args: DeepseekV4DecoderLayer.hc_post(self.layer, *args) + for setting in ( + envs.SGLANG_OPT_USE_TILELANG_MHC_POST.override(False), + envs.SGLANG_OPT_USE_FLASHINFER_MHC.override(False), + ): + setting.__enter__() + self.addCleanup(setting.__exit__, None, None, None) + torch.manual_seed(39186) + + def operands(self, rows, width=5120, dtype=torch.bfloat16): + return ( + torch.randn(rows, width, device="cuda", dtype=dtype), + torch.randn(rows, 4, width, device="cuda", dtype=dtype), + torch.sigmoid(torch.randn(rows, 4, device="cuda")), + torch.softmax(torch.randn(rows, 4, 4, device="cuda"), -1), + ) + + def test_model_dispatch_and_graph_replay(self): + from aiter.ops.mhc import mhc_post + + for rows in (1024, 1025): + with self.subTest(rows=rows): + args = self.operands(rows) + reference = torch.empty_like(args[1]) + actual = self.run_post(*args) + mhc_post(reference, *args) + torch.testing.assert_close(actual, reference, atol=0, rtol=0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = self.run_post(*args) + for _ in range(2): + args[0].normal_() + args[1].normal_() + args[2].uniform_() + args[3].uniform_() + graph.replay() + mhc_post(reference, *args) + torch.testing.assert_close(actual, reference, atol=0, rtol=0) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_dsv41_candidate_graph_variants.py b/test/registered/unit/layers/test_dsv41_candidate_graph_variants.py new file mode 100644 index 0000000000..5c9ef7d32d --- /dev/null +++ b/test/registered/unit/layers/test_dsv41_candidate_graph_variants.py @@ -0,0 +1,91 @@ +"""DeepSeek-V4.1 candidate graph variants are admitted on ROCm and bound every HIP verify position.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.attention.graph_variants import ( + create_dsv41_candidate_graph_variants, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3, suite="base-a-test-cpu") + + +class TestVerifyCandidateGraph(CustomTestCase): + def test_hip_verify_shortcut_bounds_all_draft_positions(self): + """A verify row whose last draft position passes the candidate span must not take the shortcut.""" + from sglang.srt.layers.attention.dsv4.low_ratio_backend_hip import ( + low_ratio_decode_rows_fit_candidate_span, + ) + + backend = SimpleNamespace( + low_ratio_candidate_span=16384, speculative_num_draft_tokens=6 + ) + for length, expected in ((16378, True), (16379, False)): + batch = SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY, + seq_lens_cpu=torch.tensor([length]), + ) + self.assertEqual( + low_ratio_decode_rows_fit_candidate_span(backend, batch), expected + ) + batch.seq_lens_cpu = None + self.assertFalse(low_ratio_decode_rows_fit_candidate_span(backend, batch)) + + def test_factory_keeps_causal_indexer_for_verify(self): + """gfx950 must be admitted without a Blackwell capability; other ROCm GPUs and + pre-Blackwell CUDA stay rejected.""" + config = SimpleNamespace( + model_type="deepseek_v41", + candidate_source_layer_id=1, + candidate_topk_blocks=128, + candidate_block_size=128, + compress_ratios=[1, 2], + index_topk=2048, + ) + runner = SimpleNamespace( + model_config=SimpleNamespace(hf_text_config=config), + device="cuda", + gpu_id=0, + spec_algorithm=SimpleNamespace(is_dspark=lambda: True), + is_draft_worker=False, + ) + with ( + patch("torch.cuda.get_device_capability", return_value=(9, 5)), + patch("sglang.srt.utils.is_hip", return_value=True), + patch("sglang.srt.utils.is_gfx95_supported", return_value=True), + ): + self.assertIsNotNone( + create_dsv41_candidate_graph_variants( + runner, ForwardMode.TARGET_VERIFY, 6 + ) + ) + with ( + patch("torch.cuda.get_device_capability", return_value=(9, 4)), + patch("sglang.srt.utils.is_hip", return_value=True), + patch("sglang.srt.utils.is_gfx95_supported", return_value=False), + ): + self.assertIsNone( + create_dsv41_candidate_graph_variants( + runner, ForwardMode.TARGET_VERIFY, 6 + ) + ) + with ( + patch("torch.cuda.get_device_capability", return_value=(9, 0)), + patch("sglang.srt.utils.is_hip", return_value=False), + patch("sglang.srt.utils.is_gfx95_supported", return_value=False), + ): + self.assertIsNone( + create_dsv41_candidate_graph_variants( + runner, ForwardMode.TARGET_VERIFY, 6 + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py index 52378f5b62..6897db6454 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_fused_mhc.py @@ -287,11 +287,15 @@ class TestAmdFusedMhcNormFusedHandling(unittest.TestCase): # (norm_fused=False) -- the Triton fused post+pre contract. normed = object() layer.input_layernorm.return_value = normed + layer._input_norm = lambda *a, **k: DeepseekV4DecoderLayer._input_norm( + layer, *a, **k + ) layer.self_attn.maybe_use_decode_attn_tp.side_effect = _StopForward # Force the non-aiter (torch layernorm) branch deterministically so the # test does not depend on the runner arch and needs no real tensors. with ( + mock.patch.object(deepseek_v4, "_is_hip", False), mock.patch.object(deepseek_v4, "_use_aiter", False), mock.patch.object(deepseek_v4, "_is_gfx95_supported", False), mock.patch( @@ -359,7 +363,11 @@ class TestAmdFusedMhcNumerical(unittest.TestCase): res = (torch.randn(m, hc_mult, hidden, device=dev) * 0.02).bfloat16() post = torch.randn(m, hc_mult, device=dev) * 0.02 comb = torch.randn(m, hc_mult, hc_mult, device=dev) * 0.02 - fn = (torch.randn(hc_mult3, hc_mult * hidden, device=dev) * 0.02).bfloat16() + # AITER reads fn as FP32; a BF16 allocation would be read out of bounds. + fn = ( + torch.randn(hc_mult3, hc_mult * hidden, device=dev, dtype=torch.float32) + * 0.02 + ) scl = torch.ones(hc_mult3, device=dev) base = torch.zeros(hc_mult3, device=dev) nw = torch.ones(hidden, device=dev).bfloat16() diff --git a/test/registered/unit/models/test_deepseek_v4_amd_tp4.py b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py new file mode 100644 index 0000000000..a0846df893 --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v4_amd_tp4.py @@ -0,0 +1,454 @@ +"""DeepSeek-V4.1 TP4 collectives on gfx950: fused all-reduce + mHC post under graph replay and bit-exact Engram reconstruction.""" + +import os +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + get_tp_group, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.environ import envs +from sglang.srt.layers.engram import EngramEmbedding +from sglang.srt.runtime_context import get_forward, get_parallel, reset_context +from sglang.srt.utils import is_gfx95_supported, is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kernels.utils import multigpu_pytest_main +from sglang.test.test_utils import publish_build_topology + +register_amd_ci(est_time=60, stage="stage-c", runner_config="large-8-gpu-amd-mi35x") +pytestmark = pytest.mark.skipif( + not is_hip() or "LOCAL_RANK" not in os.environ, + reason="run through the four-GPU entry point", +) + + +@pytest.fixture(scope="module") +def group(): + """The published TP4 topology and its real groups (pynccl + custom all-reduce).""" + rank, world = int(os.environ["LOCAL_RANK"]), int(os.environ["WORLD_SIZE"]) + if not is_gfx95_supported() or world != 4: + pytest.skip("requires a gfx950 group of four GPUs") + torch.cuda.set_device(rank) + torch.cuda.set_stream(torch.cuda.Stream()) + init_distributed_environment( + world_size=world, rank=rank, local_rank=rank, backend="nccl" + ) + publish_build_topology(tp_size=world, world_rank=rank, moe_runner_backend="aiter") + initialize_model_parallel() + yield get_tp_group() + destroy_model_parallel() + destroy_distributed_environment() + reset_context() + + +# ---- fused all-reduce + mHC post ---- + + +class _Projection: + reduce_results = True + + def __init__(self, group): + self.group = group + self.skip_reduction = [] + + def __call__(self, x, *, skip_all_reduce): + self.skip_reduction.append(skip_all_reduce) + return (x if skip_all_reduce else self.group.all_reduce(x)), None + + +class _Attention: + attn_tp_size = 4 + + def __init__(self, group): + self.wo_b = _Projection(group) + self.rank = group.rank_in_group + + def maybe_use_decode_attn_tp(self, forward_batch): + return nullcontext() + + def __call__(self, *, x, **kwargs): + from sglang.srt.models.deepseek_v4 import MQALayer + + return MQALayer._project_wo_b(self, x * (self.rank + 1)) + + +def _layer(group): + from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn + + torch.manual_seed(39186) + norm_weight = torch.ones(5120, device="cuda", dtype=torch.bfloat16) + layer = SimpleNamespace( + config=SimpleNamespace(model_type="deepseek_v41"), + dsa_enable_prefill_cp=False, + hc_pre_from_prev_sublayer=False, + self_attn=_Attention(group), + mlp=SimpleNamespace(tp_size=1), + hc_mult=4, + hc_sinkhorn_iters=20, + rms_norm_eps=1e-6, + hc_eps=1e-6, + hc_attn_fn=torch.randn(24, 20480, device="cuda") * 0.01, + hc_ffn_fn=torch.randn(24, 20480, device="cuda") * 0.01, + hc_attn_scale=torch.ones(3, device="cuda"), + hc_ffn_scale=torch.ones(3, device="cuda"), + hc_attn_base=torch.zeros(24, device="cuda"), + hc_ffn_base=torch.zeros(24, device="cuda"), + post_attention_layernorm=SimpleNamespace( + weight=norm_weight, variance_epsilon=1e-6 + ), + _run_moe_ffn_dp_sync=lambda x, *args, **kwargs: x, + ) + + def input_norm(x, *, coefficients, **kwargs): + _, normalized = rmsnorm_with_sinkhorn( + x, norm_weight, 1e-6, coefficients, fake_quant=False + ) + return normalized, None + + layer._input_norm = input_norm + return layer + + +def _no_deterministic_inference(): + return patch( + "sglang.srt.runtime_context.get_exec", + return_value=SimpleNamespace( + deterministic=SimpleNamespace(enable_deterministic_inference=False) + ), + ) + + +_FUSED_MHC = "sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc" + + +def _replay_fused_and_unfused(group, run, check, *, residual, pre): + """Run eagerly and capture with the fused all-reduce off and on; yield the eager + pair (the fused path copies x into the registered buffer), then the graph pair + replayed on new inputs.""" + graphs = [] + for enabled in (False, True): + # the server hook turns the tilelang post off on ROCm at model load + with ( + envs.SGLANG_OPT_USE_TILELANG_MHC_POST.override(False), + nullcontext() + if enabled + else patch(f"{_FUSED_MHC}._can_fuse_mhc", lambda *_: False), + ): + eager = run() + graph = torch.cuda.CUDAGraph() + with group.graph_capture() as capture: + run() + with torch.cuda.graph(graph, stream=capture.stream): + output = run() + check(enabled) + graphs.append((graph, output, eager)) + yield graphs[0][2], graphs[1][2] + for _ in range(3): + residual.normal_() + pre.uniform_() + for graph, _, _ in graphs: + graph.replay() + torch.cuda.synchronize() + yield graphs[0][1], graphs[1][1] + + +@pytest.mark.parametrize( + "rows,verify", [(1, False), (8, True)], ids=["decode", "verify"] +) +def test_model_handoff_and_graph_replay(group, rows, verify): + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + forward_hc_pre_from_prev_fused_boundary, + ) + + layer = _layer(group) + residual = torch.randn(rows, 4, 5120, device="cuda", dtype=torch.bfloat16) + pre = torch.sigmoid(torch.randn(rows, 4, device="cuda")) + batch = SimpleNamespace( + forward_mode=ForwardMode.TARGET_VERIFY if verify else ForwardMode.DECODE + ) + + def run(): + _, next_pre, pending = forward_hc_pre_from_prev_fused_boundary( + layer, None, residual, None, batch, None, pre, None, True + ) + return next_pre, *pending + + def check(enabled): + assert layer.self_attn.wo_b.skip_reduction[-1] == enabled + + with get_forward().scoped(sp_active=False), _no_deterministic_inference(): + for unfused, fused in _replay_fused_and_unfused( + group, run, check, residual=residual, pre=pre + ): + for expected, actual in zip(unfused, fused): + if actual.dtype == torch.bfloat16: + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + else: + torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) + + +def _moe_stub(rank, *, dual, shared_tp1): + """A DeepseekV2MoE with fake experts (routed x * (rank + 1), shared x * 0.5), so a + done or skipped reduction shows in its output.""" + from sglang.srt.layers.moe.topk import TopKOutputFormat + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + + def experts(x, *args, **kwargs): + return x * (rank + 1) + + def gate(x, *args, **kwargs): + return x + + experts.quant_method = None + experts.moe_runner_config = SimpleNamespace(inplace=False) + gate.rocm_router_max_tokens = -1 # the split-K router never serves the fake gate + moe = DeepseekV2MoE.__new__(DeepseekV2MoE) + torch.nn.Module.__init__(moe) + vars(moe).update( + tp_size=4, + is_deepseek_v4=True, + _shared_expert_tp1=shared_tp1, + layer_id=0, + is_nextn=False, + is_hash=False, + _fuse_shared_experts_inside_sbo=False, + _fuse_finalize_all_reduce=False, + num_fused_shared_experts=0, + routed_scaling_factor=1.0, + experts=experts, + gate=gate, + alt_stream=torch.cuda.Stream(), + topk=lambda *a, **kw: SimpleNamespace(format=TopKOutputFormat.STANDARD), + _maybe_quant_moe_input_once=lambda x: None, + _should_quant_routed_input_mxfp8=lambda x: False, + _forward_shared_experts=lambda x, *a, **kw: x * 0.5, + forward=lambda x, *a, **kw: ( + moe.forward_normal_dual_stream if dual else moe.forward_normal + )(x), + ) + return moe + + +@pytest.mark.parametrize( + "rows,dual,defer,shared_tp1", + [ + (1, False, False, False), + (8, False, True, False), + (8, True, True, False), + (8, False, True, True), + ], + ids=["unfused", "deferred", "dual-stream", "shared-tp1"], +) +def test_moe_model_handoff(group, rows, dual, defer, shared_tp1): + from sglang.srt.layers.moe import MoeA2ABackend + from sglang.srt.model_executor.forward_batch_info import ForwardMode + from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( + forward_hc_pre_from_prev_fused_boundary, + ) + from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer + + layer = _layer(group) + layer.mlp = _moe_stub(group.rank_in_group, dual=dual, shared_tp1=shared_tp1) + layer._run_moe_ffn_dp_sync = lambda *a, **kw: ( + DeepseekV4DecoderLayer._run_moe_ffn_dp_sync(layer, *a, **kw) + ) + layer.hc_post = lambda *a: DeepseekV4DecoderLayer.hc_post(layer, *a) + residual = torch.randn(rows, 4, 5120, device="cuda", dtype=torch.bfloat16) + pre = torch.sigmoid(torch.randn(rows, 4, device="cuda")) + batch = SimpleNamespace(forward_mode=ForwardMode.DECODE, num_token_non_padded=None) + outcomes = [] + + def run(): + hidden, next_pre, pending = forward_hc_pre_from_prev_fused_boundary( + layer, None, residual, None, batch, None, pre, None, defer + ) + outcomes.append(pending is not None) + if pending is not None: + hidden = layer.hc_post(*pending) + return hidden, next_pre + + with ( + get_forward().scoped( + sp_active=False, fuse_mlp_allreduce=False, flashinfer_trtllm_bypass=False + ), + patch( + "sglang.srt.models.deepseek_v2.post_experts_all_reduce", + side_effect=group.all_reduce, + ) as original_reduce, + patch( + "sglang.srt.layers.moe.get_moe_a2a_backend", return_value=MoeA2ABackend.NONE + ), + patch( + "sglang.srt.models.deepseek_v4.get_moe_a2a_backend", + return_value=MoeA2ABackend.NONE, + ), + _no_deterministic_inference(), + patch( + "sglang.srt.models.deepseek_v2.get_exec", + return_value=SimpleNamespace(moe=SimpleNamespace(enable_eplb=False)), + ), + ): + + def run_counting(): + original_reduce.reset_mock() + return run() + + def check(enabled): + # the fused kernel replaces the reduction unless TP1 shared experts + # need the plain one; the post stays pending only when it was deferred + # and not fused + assert original_reduce.call_count == int(not enabled or shared_tp1) + assert outcomes[-1] == (defer and (not enabled or shared_tp1)) + + for unfused, fused in _replay_fused_and_unfused( + group, run_counting, check, residual=residual, pre=pre + ): + torch.testing.assert_close(unfused[0], fused[0], atol=0.01, rtol=0.01) + torch.testing.assert_close(unfused[1], fused[1], atol=1e-5, rtol=1e-5) + + +@pytest.mark.parametrize("flag", ["mlp_reduce_scatter", "fuse_mlp_allreduce"]) +@pytest.mark.parametrize("dual", [False, True], ids=["normal", "dual-stream"]) +def test_moe_skipped_reduction_keeps_post(group, dual, flag): + """A reduce-scattered or elsewhere-fused MoE output must not reach the fused all-reduce + + mHC post: the pending post stays pending and no collective runs.""" + from sglang.srt.layers.moe.mhc_post_fusion import ( + MhcPostFusion, + use_mhc_post_fusion, + ) + + moe = _moe_stub(0, dual=dual, shared_tp1=False) + x = torch.ones(1, 5120, device="cuda", dtype=torch.bfloat16) + residual = torch.randn(1, 4, 5120, device="cuda", dtype=torch.bfloat16) + post = torch.rand(1, 4, device="cuda") + state = MhcPostFusion(residual, post, torch.rand(1, 4, 4, device="cuda"), None) + with ( + get_forward().scoped(**{flag: True}, flashinfer_trtllm_bypass=False), + use_mhc_post_fusion(state), + patch( + "sglang.srt.distributed.communication_op.tensor_model_parallel_all_reduce" + ) as reduction, + patch( + "sglang.srt.distributed.communication_op.moe_tensor_model_parallel_all_reduce" + ) as moe_reduction, + patch( + "sglang.srt.models.deepseek_v2.get_exec", + return_value=SimpleNamespace(moe=SimpleNamespace(enable_eplb=False)), + ), + ): + actual = moe(x) + assert state.output is None + reduction.assert_not_called() + moe_reduction.assert_not_called() + torch.testing.assert_close(actual, x * 1.5, atol=0, rtol=0) + + +# ---- sharded Engram ---- + + +def _engram_table(rows): + weight = torch.ones(rows, 128, dtype=torch.float32) + weight[:, 1::4] = -1 + weight[:, 2::4] = 0.5 + weight[:, 3::4] = -0.0 + weight = weight.to(torch.float8_e4m3fn) + scale = torch.tensor([0, 1, 120, 140], dtype=torch.uint8).repeat(rows, 1) + reference = ( + ( + weight.float().reshape(rows, 4, 32) + * scale.view(torch.float8_e8m0fnu).float().unsqueeze(-1) + ) + .flatten(1) + .to(torch.bfloat16) + ) + with ( + envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.override(False), + torch.device("cuda"), + ): + embed = EngramEmbedding(rows, 128, layer_id=1) + embed.weight.weight_loader(embed.weight, weight) + embed.scale.weight_loader(embed.scale, scale.view(torch.float8_e8m0fnu)) + embed.finish_load(label="test") + return embed, reference + + +@pytest.mark.parametrize("rows", [1, 17], ids=["1", "17"]) +def test_engram_eager_and_graph_reconstruction(group, rows): + """Reconstructing a sharded row must retain signed zero and BF16 subnormals.""" + embed, reference = _engram_table(rows) + ids = torch.arange(16, device="cuda", dtype=torch.int64).view(-1, 1) % rows + # plain TP (attn_dp_size == 1): the all-reduce path + eager = embed(ids) + assert torch.equal( + eager.cpu().view(torch.int16), reference[ids.cpu()].view(torch.int16) + ) + graph = torch.cuda.CUDAGraph() + with group.graph_capture() as capture: + embed(ids) + with torch.cuda.graph(graph, stream=capture.stream): + output = embed(ids) + for shift in (1, 3): + ids.add_(shift).remainder_(rows) + graph.replay() + assert torch.equal( + output.cpu().view(torch.int16), reference[ids.cpu()].view(torch.int16) + ) + + +@pytest.mark.parametrize("scatter", [False, True], ids=["False", "True"]) +def test_engram_dp_shard_reconstruction(group, scatter): + """DP distribution must preserve the owning shard's BF16 payload bits.""" + embed, reference = _engram_table(17) + rank, world = group.rank_in_group, group.world_size + ids = torch.tensor([[rank]], device="cuda", dtype=torch.int64) + batch = SimpleNamespace(dp_padding_mode=SimpleNamespace(is_max_len=lambda: scatter)) + + # Supply a fixed one-token-per-rank DP schedule; exercise real collectives. + def gather(dst, src, _batch): + group.all_gather_into_tensor(dst, src) + + # Present the plain TP groups to the layer and the DP helpers as a TP-wide + # attention-DP layout (one attention rank per GPU). + real = get_parallel() + dp_layout = SimpleNamespace( + tp_size=world, + tp_rank=rank, + tp_group=real.tp_group, + attn_tp_group=real.tp_group, + attn_dp_size=world, + attn_dp_rank=rank, + attn_tp_size=1, + attn_tp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + ) + with ( + patch("sglang.srt.layers.engram.get_global_dp_buffer_len", return_value=world), + patch("sglang.srt.layers.engram.get_parallel", return_value=dp_layout), + patch("sglang.srt.layers.dp_attention.get_parallel", return_value=dp_layout), + patch("sglang.srt.layers.engram.dp_gather_replicate", side_effect=gather), + patch( + "sglang.srt.layers.dp_attention.get_dp_local_info", + return_value=( + torch.tensor(rank, device="cuda", dtype=torch.int32), + torch.tensor(1, device="cuda", dtype=torch.int32), + ), + ), + ): + output = embed._dp_sharded_lookup(ids, batch) + assert torch.equal( + output.cpu().view(torch.int16), reference[ids.cpu()].view(torch.int16) + ) + + +if __name__ == "__main__": + multigpu_pytest_main(__name__, __file__, num_gpus=(4,)) diff --git a/test/registered/unit/models/test_deepseek_v4_amd_wo_a_bf16.py b/test/registered/unit/models/test_deepseek_v4_amd_wo_a_bf16.py index f6f8cc3b34..b3e556a219 100644 --- a/test/registered/unit/models/test_deepseek_v4_amd_wo_a_bf16.py +++ b/test/registered/unit/models/test_deepseek_v4_amd_wo_a_bf16.py @@ -30,12 +30,13 @@ test and only imported behind ``is_hip()`` -- matching the existing AMD aiter op tests. """ +import functools import unittest from unittest import mock import torch -from sglang.srt.utils.common import is_hip +from sglang.srt.utils.common import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") @@ -199,5 +200,105 @@ class TestWoABf16BatchedGemm(unittest.TestCase): self.assertLessEqual(rel, 5e-4) +@unittest.skipUnless(is_hip() and is_gfx95_supported(), "requires gfx950") +class TestWoABf16PrefillAndVerifyRoutes(unittest.TestCase): + """gfx950 prefill rows write the token-major layout directly and verify rows keep the + GEMV / split-K / strided-bmm regimes bit-exact under graph replay.""" + + def setUp(self): + from sglang.srt.models.deepseek_v4 import _apply_wo_a_bf16_matmul + from sglang.srt.runtime_context import get_context + + override = get_context().override_server_args() + override.install() + self.addCleanup(override.restore) + # the gfx950 routes are V4.1's + self.project = functools.partial(_apply_wo_a_bf16_matmul, fast_path=True) + torch.manual_seed(39186) + + def operands(self, rows, *, strided=False, dtype=torch.bfloat16, width=4096): + x = torch.randn(rows, 4 if strided else 2, width, device="cuda", dtype=dtype) + if strided: + x = x[:, 1:3] + w = torch.randn(2, 1024, width, device="cuda", dtype=dtype) * 0.015625 + return x, w + + def test_prefill_and_mutable_graph(self): + for rows, strided in ( + (4096, False), + (4097, True), + (65536, False), + ): + with self.subTest(rows=rows, strided=strided): + x, w = self.operands(rows, strided=strided) + y = self.project(x, w, is_decode=False, is_prefill=True) + self.assertTrue(y.is_contiguous()) + torch.testing.assert_close( + y, torch.einsum("tgd,grd->tgr", x, w), atol=0, rtol=0 + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + y = self.project(x, w, is_decode=False, is_prefill=True) + for _ in range(2): + x.normal_() + w.normal_(std=0.015625) + graph.replay() + torch.testing.assert_close( + y, torch.einsum("tgd,grd->tgr", x, w), atol=0, rtol=0 + ) + del graph, x, w, y + + def test_decode_verify_and_mutable_graph(self): + + for rows in (1, 2, 8, 64, 129): + with self.subTest(rows=rows): + x, w = self.operands(rows, strided=rows == 8) + kwargs = dict(is_decode=True, is_target_verify=rows > 1) + self.project(x, w, **kwargs) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + y = self.project(x, w, **kwargs) + for _ in range(2): + x.normal_() + w.normal_(std=0.015625) + graph.replay() + ref = torch.einsum("tgd,grd->tgr", x, w) + self.assertTrue(y.is_contiguous()) + if rows > 8: + torch.testing.assert_close(y, ref, atol=0, rtol=0) + else: + # split-K sums in its own fixed order: occasional one-ulp bf16 + # flips against einsum, far below one ulp (2^-8) on average + error = (y.float() - ref.float()).square().mean() + self.assertLess( + (error / ref.float().square().mean()).sqrt().item(), 1e-3 + ) + + def test_dsv4_keeps_the_main_routes(self): + """Without fast_path (DSv4) no gfx950 route runs; the aiter reroute is off by + default, so decode and verify rows take the einsum.""" + from sglang.srt.models import deepseek_v4 as dsv4 + + taken = mock.Mock(side_effect=AssertionError("DSv4 took a V4.1 gfx950 route")) + with ( + mock.patch.object(dsv4, "wo_a_bf16_gemv", taken), + mock.patch.object(dsv4, "wo_a_bf16_small_batch", taken), + mock.patch.object(dsv4._hip, "wo_a_fp8_grid_matmul", taken), + mock.patch.object(dsv4, "_wo_a_aiter_batched_gemm_enabled", False), + ): + for rows, kwargs in ( + (1, dict(is_decode=True)), + (8, dict(is_decode=True, is_target_verify=True)), + ): + with self.subTest(rows=rows): + x, w = self.operands(rows) + torch.testing.assert_close( + dsv4._apply_wo_a_bf16_matmul(x, w, **kwargs), + torch.einsum("tgd,grd->tgr", x, w), + atol=0, + rtol=0, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py index 9f1bf50574..a1a4b17b5e 100644 --- a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py +++ b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py @@ -116,6 +116,14 @@ class _Harness(deepseek_v4.MQALayer): return q_out, k_nope_out +_NO_HIP_GLUE = SimpleNamespace( + skip_head_pad=lambda attn: False, + attention_inv_rope=lambda *args, **kwargs: None, + wo_a_emits_fp8_grid=lambda attn: False, + wo_a_fp8_grid_matmul=lambda o, wo_a, fp8_grid: None, +) + + def _run(fp8, mode=ForwardMode.DECODE, cp=False, fused_verify=True): layer = _Harness() layer.dsa_enable_prefill_cp = cp @@ -141,7 +149,10 @@ def _run(fp8, mode=ForwardMode.DECODE, cp=False, fused_verify=True): patch.object(deepseek_v4, "fused_rope_inplace", return_value=None), patch.object(deepseek_v4, "_FP8_WO_A_GEMM", False), patch.object(deepseek_v4, "_is_gfx942_supported", False), + patch.object(deepseek_v4, "_is_gfx95_supported", False), patch.object(deepseek_v4, "_is_hip", True), + # the ROCm model glue is only imported on ROCm; the unified path needs none of it + patch.object(deepseek_v4, "_hip", _NO_HIP_GLUE), patch.object(deepseek_v4, "_is_npu", False), ): layer.forward(