dsv4.1-amd: serve DeepSeek-V4.1 on gfx950 (#41308)

Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
This commit is contained in:
Kevin Mi
2026-09-30 11:47:14 -07:00
committed by GitHub
co-authored by Kevin Mi
parent 9f9f9e702a
commit 3a398442bf
39 changed files with 5723 additions and 365 deletions
@@ -26,7 +26,12 @@ def _mhc_post_split_h_kernel(X, R, P, C, Y, H: tl.constexpr, B: tl.constexpr):
def mhc_post_split_h(
x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor
x: torch.Tensor,
residual: torch.Tensor,
post: torch.Tensor,
comb: torch.Tensor,
*,
block_size: int | None = None,
) -> torch.Tensor:
"""Same result as the TileLang post kernel for contiguous BF16 HC=4 inputs."""
assert x.dtype == residual.dtype == torch.bfloat16
@@ -34,7 +39,7 @@ def mhc_post_split_h(
assert residual.shape == (x.shape[0], 4, x.shape[1])
assert all(t.is_contiguous() for t in (x, residual, post, comb))
output = torch.empty_like(residual)
block = 128 if x.shape[0] <= 8 else 1024
block = block_size if block_size is not None else (128 if x.shape[0] <= 8 else 1024)
_mhc_post_split_h_kernel[(x.shape[0], triton.cdiv(x.shape[1], block))](
x,
residual,
@@ -482,8 +482,10 @@ def _block_quant_stack_applies(*, wkv_linears: list[torch.nn.Module]) -> bool:
if not (block_quant and hasattr(quant_method, "w8a8_block_fp8_linear")):
return False
block_out = quant_method.quant_config.weight_block_size[0]
# the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout
return all(
linear.weight.dtype == torch.float8_e4m3fn
and linear.weight.dim() == 2
and linear.weight.shape[0] % block_out == 0
for linear in wkv_linears
)
@@ -494,7 +496,8 @@ def _dequant_supported(linear: torch.nn.Module) -> bool:
weight = linear.weight
if weight.dtype in (torch.bfloat16, torch.float16, torch.float32):
return True
if weight.dtype != torch.float8_e4m3fn:
# the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout
if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2:
return False
block = 128
out_dim, in_dim = weight.shape
@@ -12,7 +12,7 @@ from sglang.srt.arg_groups.overrides import (
)
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import is_npu
from sglang.srt.utils.common import is_gfx95_supported, is_npu
if TYPE_CHECKING:
from sglang.srt.server_args import ServerArgs
@@ -182,7 +182,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
from sglang.srt.model_executor.cuda_graph_config import Backend
incompatible = (
("non-CUDA hardware", not get_platform().is_cuda),
(
"hardware other than CUDA or gfx950",
not (
get_platform().is_cuda
or (get_platform().is_hip and is_gfx95_supported())
),
),
(
"prefill CUDA graphs",
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
+1 -1
View File
@@ -92,7 +92,7 @@ class ExecFeatures(msgspec.Struct):
enable_encoder_swa_bounded_replay: A[
bool,
"DeepSeek-V4.1 encoder SWA bounded replay: cache Main KV and Indexer keys only, "
"rebuild request-owned SWA windows on prefix hits. Experimental; CUDA only.",
"rebuild request-owned SWA windows on prefix hits. Experimental; CUDA and gfx950.",
] = False
enable_decoder_swa_bounded_replay: A[
bool,
+3 -1
View File
@@ -922,7 +922,9 @@ class Envs:
# output columns ride along nearly free.
SGLANG_ROCM_K3_FUSE_KDA_INPROJ = EnvBool(True)
SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256)
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang")
# ROCm decode attention kernel: auto (aiter_sparse on gfx950, tilelang elsewhere) |
# aiter_sparse | tilelang | triton | torch | comparison | unified_kv_triton
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("auto")
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
@@ -2933,6 +2933,15 @@ class DeepseekV4AttnBackend(
# out_loc is -1 for an incomplete group and 0 for padding;
# the kernel suppresses both stores.
assert out_loc is not None
if pool.low_ratio_index_k_is_split(layer_id):
# ROCm keeps the index-K payload and scales in two buffers
from sglang.srt.layers.attention.dsv4.low_ratio_backend_hip import (
store_index_k_split,
)
return store_index_k_split(
pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id
)
index_k_norm_rope_pack_store(
indexer.forward_wk(latent),
indexer.k_norm.weight.data,
File diff suppressed because it is too large Load Diff
@@ -16,19 +16,31 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import (
fake_quant_fp4,
)
from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens
from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.utils import add_prefix
from sglang.srt.utils import add_prefix, is_gfx95_supported
def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False):
if x.is_cuda and torch.version.cuda is not None and x.dtype == torch.bfloat16:
def _fused_dsv41_kernels(x: torch.Tensor) -> bool:
# validated on CUDA and gfx950; other ROCm GPUs keep the torch forms
return x.is_cuda and (torch.version.cuda is not None or is_gfx95_supported())
def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False, positions=None):
"""RoPE plus fake FP4 quantization, fused for bf16 inputs on CUDA and ROCm. With
positions, freqs is the whole table and the fused kernel gathers freqs[positions]."""
if _fused_dsv41_kernels(x) and x.dtype == torch.bfloat16:
from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
rope_tail_fake_quant_fp4,
)
return rope_tail_fake_quant_fp4(x, freqs, rope_dim, compressed_kv=compressed_kv)
return rope_tail_fake_quant_fp4(
x, freqs, rope_dim, compressed_kv=compressed_kv, positions=positions
)
if positions is not None:
freqs = freqs[positions]
quant = fake_quant_compressed_kv if compressed_kv else fake_quant_fp4
return quant(rope_tail(x, freqs, rope_dim))
@@ -42,9 +54,9 @@ class RMSNorm(nn.Module):
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
# the Triton kernel is bitwise this fallback for bf16 rows on CUDA and ROCm (gfx950)
if (
x.is_cuda
and torch.version.cuda is not None
_fused_dsv41_kernels(x)
and x.dtype in (torch.bfloat16, torch.float32)
and self.weight.dtype in (torch.bfloat16, torch.float32)
and x.shape[-1] in (128, 512)
@@ -87,11 +99,13 @@ def rope_tail(
def fused_low_ratio_compress_supported() -> bool:
"""The fused c1 / c2 / index-K decode kernels pack fp4 with
`cvt.rn.satfinite.e2m1x2`, an sm100+ instruction; the answer also fixes the
"""The fused c1 / c2 / index-K decode kernels need sm100+ on CUDA (they pack fp4
with `cvt.rn.satfinite.e2m1x2`) or gfx950 on ROCm; the answer also fixes the
ratio-2 weight layout (`wkv_gate`, or `wkv` plus `wgate`)."""
if not torch.cuda.is_available() or torch.version.hip is not None:
if not torch.cuda.is_available():
return False
if torch.version.hip is not None:
return is_gfx95_supported()
return torch.cuda.get_device_capability()[0] >= 10
@@ -208,6 +222,10 @@ class DeepseekV41Indexer(nn.Module):
quant_config=None,
prefix=add_prefix("weights_proj", prefix),
)
# -1 off gfx950 or for shapes the split-K GEMV lacks
self.weights_proj_hip_max_tokens = rocm_gemv_split_k_max_tokens(
n=self.n_heads, k=config.hidden_size, weight_dtype=torch.bfloat16
)
# The decode GEMM matches tiny_gemm's reduction order, not cuBLAS's.
self.weights_proj_small_max_m = _small_weights_proj_max_m(
self.n_heads, config.hidden_size
@@ -236,10 +254,17 @@ class DeepseekV41Indexer(nn.Module):
k = self.k_norm(self.forward_wk(latent))
return _rope_fq4(k, freqs, self.rope_head_dim)
def queries(self, q_lora: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
def queries(
self,
q_lora: torch.Tensor,
freqs: torch.Tensor,
positions: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""freqs per token, or the whole table with positions (gathered inside the
fused RoPE launch)."""
q, _ = self.wq_b(q_lora)
q = q.view(q.shape[0], self.n_local_heads, self.index_head_dim)
return _rope_fq4(q, freqs, self.rope_head_dim)
return _rope_fq4(q, freqs, self.rope_head_dim, positions=positions)
def head_weights_raw(self, x: torch.Tensor) -> torch.Tensor:
"""`weights_proj(x)` before the scale, [tokens, n_heads] bf16."""
@@ -0,0 +1,611 @@
"""DeepSeek V4.1 low-ratio (1 / 2) indexer on ROCm: FlyDSL fp4 paged MQA logits over the split
payload / scale index-K pools, then the AOT top-k transform -- the DeepGEMM path's contract."""
from __future__ import annotations
import itertools
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
import torch
import triton
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
CandidateBlocks,
cat_candidate_blocks,
select_candidate_blocks_hip,
slice_candidate_blocks,
topk_transform_paged_sorted,
topk_within_candidate_blocks_hip,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
LOW_RATIO_PAGE_TABLE_BUCKET,
FP4DecodeWorkspace,
FP4PrefillWorkspace,
aiter_fp4_paged_mqa_logits,
index_q_rope_pack_weights_flydsl,
indexer_head_weights,
logits_rows_per_chunk,
pack_fp4_query_flydsl,
prepare_fp4_decode_workspace,
prepare_fp4_prefill_workspace,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import (
index_k_norm_rope_pack_store_split,
)
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
from sglang.srt.layers.attention.deepseek_v4_backend import (
_as_int_list,
)
if TYPE_CHECKING:
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
# fp32 logits scored per candidate-block chunk stay under this many bytes
_CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30
def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool:
"""Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch."""
return (
0 < x.shape[0] <= indexer.weights_proj_hip_max_tokens
and x.dim() == 2
and x.dtype == torch.bfloat16
and x.stride(1) == 1
)
def _indexer_head_weights(indexer, x: torch.Tensor) -> torch.Tensor:
"""indexer.head_weights(x) as the contiguous bf16 [T, H] the FlyDSL kernels take; decode row
counts run indexer_head_weights (same two roundings as aiter's GEMM plus the scale)."""
if _gemv_head_weight_rows(indexer, x):
return indexer_head_weights(
x, indexer.weights_proj.weight, indexer.head_weight_scale
)
return indexer.head_weights(x).contiguous()
def _indexer_inputs(layer, x, q_lora, pos):
"""(payload, scale) query in the FlyDSL layout and the pre-scaled head weights."""
indexer = layer.indexer
# The kernel sums head scores locally, so the indexer heads must be replicated.
assert indexer.n_local_heads == indexer.n_heads
if (
_gemv_head_weight_rows(indexer, x)
and indexer.n_heads % 16 == 0
and indexer.n_heads <= 64
and indexer.index_head_dim == 128
and layer.freqs_cis.dtype == torch.complex64
):
# decode rows: wq_b, split-K head-weight GEMV, then one launch for RoPE, fp4 pack and reduce
q, _ = indexer.wq_b(q_lora)
partials = rocm_router_gemv_split_k(x, indexer.weights_proj.weight)
return index_q_rope_pack_weights_flydsl(
q,
layer.freqs_cis,
pos,
indexer.rope_head_dim,
partials,
indexer.head_weight_scale,
num_heads=indexer.n_heads,
)
# [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself
q = indexer.queries(q_lora, layer.freqs_cis, positions=pos)
q_fp4, q_scale = pack_fp4_query_flydsl(q)
weights = _indexer_head_weights(indexer, x) # [T, H] bf16, already scaled
return q_fp4, q_scale, weights
def build_low_ratio_decode_workspaces(
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
) -> Dict[int, FP4DecodeWorkspace]:
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace."""
return {
ratio: prepare_fp4_decode_workspace(
meta.page_table,
meta.compressed_seq_lens,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
)
for ratio, meta in metadata_by_ratio.items()
}
def refresh_low_ratio_prefill_workspaces(
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
previous: Optional[Dict[int, FP4PrefillWorkspace]],
) -> Dict[int, FP4PrefillWorkspace]:
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace."""
previous = previous or {}
return {
ratio: prepare_fp4_prefill_workspace(
meta.page_table,
meta.compressed_seq_lens,
workspace=previous.get(ratio),
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
)
for ratio, meta in metadata_by_ratio.items()
}
def low_ratio_identity_skip_enabled(
*, index_topk: int, candidate_topk_blocks: int, candidate_block_size: int
) -> bool:
"""Whether identity requests may bypass scoring under this model config: the score-free
candidate mask is right only while <= index_topk positions fit the candidate top-k."""
return triton.cdiv(index_topk, candidate_block_size) <= candidate_topk_blocks
def _fill_identity_request(
*,
lc: int,
slots_j: torch.Tensor,
lens: torch.Tensor,
page_rows: torch.Tensor,
raw_rows: Optional[torch.Tensor],
) -> None:
"""Rows [t, topk] of the -1 filled buffers: column j < lc holds compressed
position j (its slot / its raw index) when j < lens_row, else -1. An identity
request publishes no candidates: every reachable block is one."""
j = torch.arange(lc, device=lens.device)
reach = j[None, :] < lens[:, None]
page_rows[:, :lc] = torch.where(reach, slots_j[None, :], -1).to(torch.int32)
if raw_rows is not None:
raw_rows[:, :lc] = torch.where(reach, j[None, :], -1).to(torch.int32)
def _decode_batch_max_seq_len(forward_batch) -> Optional[int]:
"""The host-side longest context of a decode batch, None when unknown."""
seq_lens_cpu = forward_batch.seq_lens_cpu
if seq_lens_cpu is None or seq_lens_cpu.numel() == 0:
return None
return int(seq_lens_cpu.max().item())
def low_ratio_decode_rows_are_identity(backend, forward_batch, ratio: int) -> bool:
"""Decode: every row's compressed context fits index_topk, so nothing needs scoring. Inside a
captured graph the answer is the variant being captured; eagerly it is the host-side batch
maximum. Target-verify rows keep the scored path."""
if not backend.low_ratio_identity_skip or forward_batch is None:
return False
if not forward_batch.forward_mode.is_decode():
return False
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_is_capture_mode,
skip_low_ratio_indexer,
)
if get_is_capture_mode():
return skip_low_ratio_indexer(ratio)
max_len = _decode_batch_max_seq_len(forward_batch)
return max_len is not None and max_len // ratio <= backend.index_topk
def low_ratio_candidate_span(hf_text_config) -> Optional[int]:
"""The span (in positions) within which every block of a request is a
candidate, when the model has a candidate source; None otherwise."""
# optional HF-config keys: a model without a candidate source lacks them
if getattr(hf_text_config, "candidate_source_layer_id", -1) < 0:
return None
span = getattr(hf_text_config, "candidate_topk_blocks", 0) * getattr(
hf_text_config, "candidate_block_size", 0
)
return span if span > 0 else None
def low_ratio_decode_rows_fit_candidate_span(backend, forward_batch) -> bool:
"""Decode: every request's context fits the candidate span, so two-level top-k equals the plain
paged top-k. Captured: the variant being captured; eager: the host-side batch maximum."""
span = backend.low_ratio_candidate_span
if span is None or forward_batch is None:
return False
if not (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()
):
return False
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_capture_attention_variant,
get_is_capture_mode,
)
if get_is_capture_mode():
return get_capture_attention_variant() in (
"candidate_all",
"candidate_c2_all",
"candidate_unfiltered",
)
max_len = _decode_batch_max_seq_len(forward_batch)
if forward_batch.forward_mode.is_target_verify():
if max_len is None:
return False
max_len += backend.speculative_num_draft_tokens
return max_len is not None and max_len <= span
def low_ratio_index_topk_hip_decode(
backend, layer, x, q_lora, pos, forward_batch: ForwardBatch
) -> None:
"""One token per request: paged fp4 logits over every visible compressed slot, level-one
candidate blocks where the layer publishes or consumes them, then the top-k transform
writes the -1 padded page indices."""
pool = backend.token_to_kv_pool
metadata = backend.forward_metadata
core = metadata.core_metadata
ratio = layer.compress_ratio
indexer = layer.indexer
indexer_metadata = metadata.low_ratio_indexer_metadata(ratio)
assert indexer_metadata is not None, f"no decode indexer metadata for {ratio = }"
page_indices = core.sparse_page_indices(ratio)
raw_indices = core.sparse_raw_indices(ratio)
if low_ratio_decode_rows_are_identity(backend, forward_batch, ratio):
# every row takes the transform's sequential branch (len <= topk), which reads no scores
scores = torch.empty(
(page_indices.shape[0], 1), dtype=torch.float32, device=pos.device
)
topk_transform_paged(
scores,
indexer_metadata.compressed_seq_lens,
indexer_metadata.page_table,
page_indices,
indexer_metadata.compressed_page_size,
raw_indices,
)
return
two_level = indexer.is_candidate_source or indexer.uses_candidates
if two_level and low_ratio_decode_rows_fit_candidate_span(backend, forward_batch):
# every block is a candidate: the source publishes nothing, every layer runs the plain top-k
if indexer.is_candidate_source:
backend.candidate_masks = None
two_level = False
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
logits = aiter_fp4_paged_mqa_logits(
q_fp4=q_fp4,
q_scale=q_scale,
k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id),
k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id),
weights=weights,
page_table=indexer_metadata.page_table,
c4_seq_lens=indexer_metadata.compressed_seq_lens,
weight_scale=1.0,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
is_decode=True,
decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio),
)
# level one is bounded on device by the compressed lengths: a captured step cannot read them back
if two_level and indexer.uses_candidates:
candidates = backend.candidate_masks
assert (
isinstance(candidates, CandidateBlocks)
and candidates.ids.shape[0] == logits.shape[0]
and candidates.block_size == indexer.candidate_block_size
), "candidate blocks missing for decode"
topk_within_candidate_blocks_hip(
logits,
indexer_metadata.compressed_seq_lens,
candidates,
page_table=indexer_metadata.page_table,
page_size=indexer_metadata.compressed_page_size,
page_indices=core.sparse_page_indices(ratio),
raw_indices=core.sparse_raw_indices(ratio),
sort_output=True,
)
return
if two_level and indexer.is_candidate_source:
backend.candidate_masks = select_candidate_blocks_hip(
logits,
indexer_metadata.compressed_seq_lens,
topk_blocks=indexer.candidate_topk_blocks,
block_size=indexer.candidate_block_size,
)
topk_transform_paged_sorted(
logits,
indexer_metadata.compressed_seq_lens,
indexer_metadata.page_table,
page_indices,
indexer_metadata.compressed_page_size,
raw_indices,
)
def _extend_k_slots(req_to_token, *, ratio, lc_per_req, req_pool_indices, device):
"""Per request, the c1/c2 pool slots of its visible compressed positions, and
each request's start offset in their concatenation."""
starts = [0, *itertools.accumulate(lc_per_req)][:-1]
counts = torch.tensor(lc_per_req, dtype=torch.int64, device=device)
req_rows = torch.repeat_interleave(req_pool_indices.to(torch.int64), counts)
j = torch.arange(int(counts.sum()), device=device) - torch.repeat_interleave(
torch.tensor(starts, dtype=torch.int64, device=device), counts
)
slots = req_to_token[req_rows, j * ratio].to(torch.int64) // ratio
return slots, starts
def low_ratio_index_topk_hip_extend(
backend,
layer,
x,
q_lora,
pos,
forward_batch: ForwardBatch,
) -> None:
"""Ragged prefill: the FlyDSL prefill kernel scores every token's visible compressed positions,
candidate masks are published or applied per request, one paged top-k selects every row.
Identity requests are written without scores; an all-identity batch skips the kernel."""
pool = backend.token_to_kv_pool
metadata = backend.forward_metadata
core = metadata.core_metadata
ratio = layer.compress_ratio
indexer = layer.indexer
page_indices = core.sparse_page_indices(ratio)
raw_indices = core.sparse_raw_indices(ratio)
page_indices.fill_(-1)
if raw_indices is not None:
raw_indices.fill_(-1)
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
# bounded replay counts cover only each request's tail
if metadata.late_layer_tail is not None:
extend_lens_cpu = metadata.late_layer_tail.extend_seq_lens_cpu
else:
extend_lens_cpu = _as_int_list(forward_batch.extend_seq_lens_cpu)
assert seq_lens_cpu is not None and extend_lens_cpu is not None
lc_per_req = [s // ratio for s in seq_lens_cpu]
if pos.numel() == 0 or not any(lc_per_req):
if indexer.is_candidate_source:
backend.candidate_masks = [None] * len(lc_per_req)
return
if backend.low_ratio_identity_skip:
# identity: every row selects all its visible compressed positions
is_identity = [s // ratio <= indexer.index_topk for s in seq_lens_cpu]
else:
is_identity = [False] * len(seq_lens_cpu)
compress_lens = ((pos + 1) // ratio).to(torch.int32)
consume = backend.candidate_masks if indexer.uses_candidates else None
publish = [] if indexer.is_candidate_source else None
if any(is_identity):
# only the score-free fill resolves slots itself; scored rows resolve in the top-k transform
k_slots, starts = _extend_k_slots(
backend.req_to_token,
ratio=ratio,
lc_per_req=lc_per_req,
req_pool_indices=forward_batch.req_pool_indices.to(torch.int64),
device=pos.device,
)
def fill_identity_requests(req_lo, req_hi, tok_lo):
"""Identity requests req_lo..req_hi: rows written without scores, None published per request."""
tok = tok_lo
for b in range(req_lo, req_hi):
t_len, lc = extend_lens_cpu[b], lc_per_req[b]
rows = slice(tok, tok + t_len)
tok += t_len
if lc == 0:
if publish is not None:
publish.append(None)
continue
_fill_identity_request(
lc=lc,
slots_j=k_slots[starts[b] : starts[b] + lc],
lens=compress_lens[rows],
page_rows=page_indices[rows],
raw_rows=None if raw_indices is None else raw_indices[rows],
)
if publish is not None:
publish.append(None)
if all(is_identity):
fill_identity_requests(0, len(extend_lens_cpu), 0)
if publish is not None:
backend.candidate_masks = publish
return
indexer_metadata = metadata.low_ratio_indexer_metadata(ratio)
assert indexer_metadata is not None, f"no prefill indexer metadata for {ratio = }"
num_tokens = pos.shape[0]
assert indexer_metadata.page_table.shape[0] >= num_tokens
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio)
def score_rows(rows: slice) -> torch.Tensor:
return aiter_fp4_paged_mqa_logits(
q_fp4=q_fp4[rows],
q_scale=q_scale[rows],
k_payload=k_payload,
k_scale=k_scale,
weights=weights[rows],
page_table=indexer_metadata.page_table[rows],
c4_seq_lens=indexer_metadata.compressed_seq_lens[rows],
weight_scale=1.0,
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
is_decode=False,
prefill_workspace=prefill_workspace if rows.start == 0 else None,
)
def select_rows(
rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish
):
_select_topk_extend_hip(
indexer=indexer,
logits=score_rows(rows),
lc_per_req=lc_per_req[req_lo:req_hi],
extend_lens_cpu=[rows.stop - rows.start]
if req_hi == req_lo + 1
else extend_lens_cpu[req_lo:req_hi],
is_identity=group_is_identity,
compress_lens=compress_lens[rows],
page_table=indexer_metadata.page_table[rows],
page_size=indexer_metadata.compressed_page_size,
page_indices=page_indices[rows],
raw_indices=raw_indices[rows] if raw_indices is not None else None,
consume=consume_rows,
publish=publish,
)
# group requests to fit the pooled logits block; rows are independent, so grouping is exact
rows_per_chunk = logits_rows_per_chunk(
indexer_metadata.page_table, LOW_RATIO_PAGE_TABLE_BUCKET
)
groups = _request_groups(extend_lens_cpu, rows_per_chunk)
for req_lo, req_hi, tok_lo, tok_hi in groups:
group_is_identity = is_identity[req_lo:req_hi]
if all(group_is_identity):
fill_identity_requests(req_lo, req_hi, tok_lo)
continue
if tok_hi - tok_lo <= rows_per_chunk:
select_rows(
slice(tok_lo, tok_hi),
req_lo,
req_hi,
group_is_identity,
None if consume is None else consume[req_lo:req_hi],
publish,
)
continue
# a request wider than the logits block: scored in row chunks, its mask published in one piece
assert req_hi == req_lo + 1, (req_lo, req_hi)
pieces = []
for lo in range(tok_lo, tok_hi, rows_per_chunk):
rows = slice(lo, min(lo + rows_per_chunk, tok_hi))
piece = [] if publish is not None else None
consume_rows = None
if consume is not None:
consume_rows = [
None
if consume[req_lo] is None
else slice_candidate_blocks(
consume[req_lo], slice(lo - tok_lo, rows.stop - tok_lo)
)
]
select_rows(rows, req_lo, req_hi, group_is_identity, consume_rows, piece)
if piece:
pieces.append(piece[0])
if publish is not None:
publish.append(cat_candidate_blocks(pieces))
if publish is not None:
backend.candidate_masks = publish
def _select_topk_extend_hip(
*,
indexer,
logits: torch.Tensor,
lc_per_req: List[int],
extend_lens_cpu: List[int],
is_identity: List[bool],
compress_lens: torch.Tensor,
page_table: torch.Tensor,
page_size: int,
page_indices: torch.Tensor,
raw_indices: Optional[torch.Tensor],
consume: Optional[List[Optional[CandidateBlocks]]],
publish: Optional[List[Optional[CandidateBlocks]]],
) -> None:
"""Row t of logits scores its request's compressed positions in columns 0..lc-1, reachable
up to compress_lens[t]. A source layer publishes one CandidateBlocks per request (None
for identity / empty), a consumer selects inside its published blocks, others run one paged top-k."""
assert page_indices.shape[1] == indexer.index_topk, (
f"the paged top-k selects page_indices.shape[1] = {page_indices.shape[1]} "
f"slots, the indexer wants {indexer.index_topk}"
)
if publish is not None:
tok_start = 0
for b, (lc, t_len) in enumerate(zip(lc_per_req, extend_lens_cpu)):
rows = slice(tok_start, tok_start + t_len)
tok_start += t_len
if lc == 0 or t_len == 0 or is_identity[b]:
# Consumers index the publication by request, so keep the slot.
publish.append(None)
continue
step = max(1, _CANDIDATE_SCORE_BUDGET_BYTES // (lc * 4))
pieces = []
for lo in range(0, t_len, step):
chunk = slice(rows.start + lo, rows.start + min(lo + step, t_len))
pieces.append(
select_candidate_blocks_hip(
logits[chunk, :lc],
compress_lens[chunk],
topk_blocks=indexer.candidate_topk_blocks,
block_size=indexer.candidate_block_size,
)
)
publish.append(cat_candidate_blocks(pieces))
if consume is not None:
tok_start = 0
for b, t_len in enumerate(extend_lens_cpu):
rows = slice(tok_start, tok_start + t_len)
tok_start += t_len
if t_len == 0:
continue
rows_page = page_indices[rows]
rows_raw = raw_indices[rows] if raw_indices is not None else None
if consume[b] is None:
topk_transform_paged_sorted(
logits[rows],
compress_lens[rows].contiguous(),
page_table[rows],
rows_page,
page_size,
rows_raw,
)
else:
topk_within_candidate_blocks_hip(
logits[rows],
compress_lens[rows],
consume[b],
page_table=page_table[rows],
page_size=page_size,
page_indices=rows_page,
raw_indices=rows_raw,
sort_output=True,
)
return
topk_transform_paged_sorted(
logits,
compress_lens.contiguous(),
page_table,
page_indices,
page_size,
raw_indices,
)
def _request_groups(
extend_lens_cpu: List[int], rows_per_chunk: int
) -> List[Tuple[int, int, int, int]]:
"""Consecutive request groups whose token rows fit rows_per_chunk (a single
request always forms a group): (req_lo, req_hi, tok_lo, tok_hi)."""
groups = []
req_lo, tok_lo, tok = 0, 0, 0
for r, t_len in enumerate(extend_lens_cpu):
if r > req_lo and tok + t_len - tok_lo > rows_per_chunk:
groups.append((req_lo, r, tok_lo, tok))
req_lo, tok_lo = r, tok
tok += t_len
groups.append((req_lo, len(extend_lens_cpu), tok_lo, tok))
return groups
def store_index_k_split(
pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id: int
) -> None:
"""The c1 / c2 decode index-K store into ROCm's split pools: payload and scales in
two buffers, where CUDA packs one fused [.., 68]-byte row."""
index_k_norm_rope_pack_store_split(
indexer.forward_wk(latent),
indexer.k_norm.weight.data,
indexer.k_norm.eps,
freqs_cis,
pos,
out_loc,
pool.get_index_k_fp4_payload_buffer(layer_id),
pool.get_index_k_fp4_scale_buffer(layer_id),
ratio=layer.compress_ratio,
)
@@ -98,7 +98,7 @@ def create_dsv41_candidate_graph_variants(
import torch
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.utils import is_hip
from sglang.srt.utils import is_gfx95_supported, is_hip
text_config = model_runner.model_config.hf_text_config
dspark_target_verify = (
@@ -110,8 +110,11 @@ def create_dsv41_candidate_graph_variants(
if not (
(capture_forward_mode == ForwardMode.DECODE or dspark_target_verify)
and model_runner.device == "cuda"
and not is_hip()
and torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10
and (
is_gfx95_supported()
if is_hip()
else torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10
)
and getattr(text_config, "model_type", None) == "deepseek_v41"
and getattr(text_config, "candidate_source_layer_id", -1) >= 0
):
@@ -1,17 +1,207 @@
from typing import Any, Optional
from typing import Any, Optional, Tuple
import torch
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
from sglang.srt.environ import envs
from sglang.srt.utils import is_hip
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_gfx95_supported, is_hip
FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
_HIP_BACKENDS = ("tilelang", "triton", "aiter_sparse", "torch", "comparison")
# at this many query tokens and above the aiter sparse kernel runs unsplit (batch-invariant prefill)
_AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS = 1024
def hip_attn_kv_splits() -> int:
"""Split-KV count for the HIP sparse decode kernels. Deterministic inference pins 4
splits (a fixed combine order keeps the bits identical at every batch size) and
everything else uses 0: adaptive splits plus the native 16-head attention for small
TP4 decode batches."""
try:
deterministic = get_exec().deterministic.enable_deterministic_inference
except ValueError: # no published exec config (kernel tests, offline tools)
deterministic = False
return 4 if deterministic else 0
# the eager warmups before each capture fill this, so captured graphs reuse these tensors
_UNIFORM_INDPTR_CACHE: dict[tuple[int, int, str], torch.Tensor] = {}
def _uniform_indptr(num_tokens: int, width: int, device: str) -> torch.Tensor:
"""Row pointers of the aiter sparse decode kernel: token t reads kv_indices[t*w : (t+1)*w].
Only tensors built outside capture are cached: an arange recorded into a graph runs only
when that graph replays, and V4.1 captures two verify graphs per batch size."""
key = (num_tokens, width, device)
indptr = _UNIFORM_INDPTR_CACHE.get(key)
if indptr is not None:
return indptr
indptr = torch.arange(
0, (num_tokens + 1) * width, width, dtype=torch.int32, device=device
)
if (
num_tokens < _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS
and not torch.cuda.is_current_stream_capturing()
):
_UNIFORM_INDPTR_CACHE[key] = indptr
return indptr
def aiter_sparse_decode_fwd(
q: torch.Tensor,
k_cache: torch.Tensor,
indices: torch.Tensor,
attn_sink: torch.Tensor,
softmax_scale: float,
extra_k_cache: Optional[torch.Tensor] = None,
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
topk_length: Optional[torch.Tensor] = None,
extra_topk_length: Optional[torch.Tensor] = None,
**_,
):
"""aiter's gfx950 gluon sparse decode kernel (pa_decode_sparse) behind the
flash_mla_with_kvcache shapes. Only -1 entries are skipped, so callers fold
topk_length into the index lists; inv_rope folds the model's inverse RoPE into the output."""
if k_cache.shape[-1] in (
KVLayout.V41.bytes_per_token,
KVLayout.V41_FP4.bytes_per_token,
):
from sglang.kernels.ops.attention.dsv4.compact_attention_hip import (
compact_attention_hip,
)
b, s, h, d = q.shape
out = compact_attention_hip(
q.reshape(b * s, h, d),
k_cache,
indices,
topk_length.reshape(-1),
attn_sink,
extra_cache=extra_k_cache,
extra_indices=extra_indices_in_kvcache,
extra_lengths=(
extra_topk_length.reshape(-1) if extra_k_cache is not None else None
),
softmax_scale=softmax_scale,
inv_rope=inv_rope,
splits=(
1
if b * s >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS
else hip_attn_kv_splits() or None
),
)
return out.view(b, s, h, d), None
from aiter.ops.triton.attention.pa_decode_sparse import pa_decode_sparse
from sglang.kernels.ops.attention.sparse_decode_reduce_hip import (
gfx_sparse_split_reduce,
)
b, s, h, d = q.shape
n = b * s
q3 = q.reshape(n, h, d)
assert k_cache.dim() == 4 and k_cache.shape[2] == 1, k_cache.shape
cache = k_cache.view(torch.uint8).squeeze(2)
idx = indices.reshape(-1)
assert idx.dtype == torch.int32 and idx.is_contiguous()
indptr = _uniform_indptr(n, indices.shape[-1], str(q.device))
extra_kwargs = {}
if extra_k_cache is not None:
assert extra_k_cache.dim() == 4 and extra_k_cache.shape[2] == 1
extra_idx = extra_indices_in_kvcache.reshape(-1)
assert extra_idx.dtype == torch.int32 and extra_idx.is_contiguous()
extra_kwargs = dict(
extra_cache=extra_k_cache.view(torch.uint8).squeeze(2),
extra_indices=extra_idx,
extra_indptr=_uniform_indptr(
n, extra_indices_in_kvcache.shape[-1], str(q.device)
),
)
if n >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS:
extra_kwargs["kv_splits"] = 1
elif hip_attn_kv_splits() > 0:
# a pinned split count keeps the combine order, hence the bits, the same at every batch size
extra_kwargs["kv_splits"] = hip_attn_kv_splits()
out = pa_decode_sparse(
q3,
cache,
idx,
indptr,
attn_sink,
softmax_scale,
skip_reduce=True,
**extra_kwargs,
)
if isinstance(out, tuple):
# Split-KV partials (acc, m, l): combine them here.
part_acc, part_m, part_l = out
out = gfx_sparse_split_reduce(
part_acc, part_m, part_l, attn_sink, q.dtype, inv_rope=inv_rope
)
elif inv_rope is not None:
_apply_inverse_rope(out, inv_rope)
return out.view(b, s, h, d), None
def _apply_inverse_rope(
out: torch.Tensor, inv_rope: Tuple[torch.Tensor, torch.Tensor]
) -> None:
"""The model's standalone inverse RoPE on out [n, h, d] (last 64 dims of
every head), for the paths that did not fold it into their combine."""
from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace
freqs_real, positions = inv_rope
n = out.shape[0]
# freqs_real is view_as_real(freqs_cis).flatten(-2); fused_rope_inplace takes the complex table
freqs_cis = torch.view_as_complex(freqs_real.view(freqs_real.shape[0], -1, 2))
fused_rope_inplace(
out.view(n, -1, out.shape[-1])[..., -freqs_real.shape[-1] :],
None,
freqs_cis,
positions.view(-1)[:n],
inverse=True,
)
def resolve_hip_flashmla_backend(backend: Optional[str] = None) -> str:
"""The HIP decode attention kernel name; "auto" (the default) is aiter's
gluon sparse kernel on gfx950 and the tilelang partial + combine elsewhere."""
if backend is None:
backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get()
if backend == "auto":
return "aiter_sparse" if is_gfx95_supported() else "tilelang"
if backend == "aiter_sparse" and not is_gfx95_supported():
raise ValueError(
"SGLANG_HACK_FLASHMLA_BACKEND=aiter_sparse needs gfx950 (aiter's gluon sparse "
"decode kernel); use auto or tilelang"
)
return backend
# kernels taking the real per-rank head count; the tilelang kernel builds only for 64-padded widths
_HIP_BACKENDS_ANY_HEAD_COUNT = frozenset({"aiter_sparse", "triton"})
def hip_attention_needs_head_pad() -> bool:
"""Whether the kernel DeepseekV4HipRadixBackend.forward picks needs the per-rank query
heads padded to 64 (zero q, zero sink)."""
return resolve_hip_flashmla_backend() not in _HIP_BACKENDS_ANY_HEAD_COUNT
def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
if is_hip():
backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get()
# a CUDA backend name (the "comparison" recursion passes "kernel") takes the HIP default
backend = resolve_hip_flashmla_backend(
backend if backend in _HIP_BACKENDS else None
)
else:
import sgl_kernel.flash_mla as flash_mla
@@ -44,6 +234,9 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
return triton_fp8_attention_fwd(**kwargs)
if backend == "aiter_sparse":
return aiter_sparse_decode_fwd(**kwargs)
if backend == "kernel":
return flash_mla.flash_mla_with_kvcache(**kwargs)
+24 -6
View File
@@ -33,6 +33,7 @@ from sglang.kernels.ops.embeddings.engram_hash import (
engram_hash_ids_and_commit,
)
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.distributed.parallel_state import inplace_all_reduce
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
attn_cp_all_gather_into_tensor,
@@ -47,7 +48,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.runtime_context import get_model, get_parallel, get_serving
from sglang.srt.utils import add_prefix, is_cuda
from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip
from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer
logger = logging.getLogger(__name__)
@@ -55,10 +56,13 @@ logger = logging.getLogger(__name__)
_MILLER_RABIN_WITNESSES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37)
_is_hip = is_hip()
def _cuda_kernels(t: torch.Tensor) -> bool:
"""True where the Triton kernels apply; ROCm and CPU take the torch paths."""
return t.is_cuda and is_cuda()
"""True where the Triton kernels apply (CUDA and gfx950); other ROCm GPUs and CPU
take the torch paths."""
return t.is_cuda and (is_cuda() or is_gfx95_supported())
def _is_prime(n: int) -> bool:
@@ -775,9 +779,18 @@ class EngramEmbedding(nn.Module):
return self._empty(indices)
values = self._owned_rows(indices)
if self.tp_size > 1:
values = tensor_model_parallel_all_reduce(values)
values = self._reduce_owned_rows(values)
return values
def _reduce_owned_rows(self, values: torch.Tensor) -> torch.Tensor:
if _is_hip and values.is_cuda:
# Integer addition preserves all BF16 bits because exactly one shard owns each row.
inplace_all_reduce(
values.view(torch.int32), group_name=get_parallel().tp_group.unique_name
)
return values
return tensor_model_parallel_all_reduce(values)
def _empty(self, indices: torch.Tensor) -> torch.Tensor:
return torch.empty(
*indices.shape, self.dim, dtype=torch.bfloat16, device=indices.device
@@ -833,9 +846,14 @@ class EngramEmbedding(nn.Module):
and self.tp_size == get_parallel().attn_dp_size
and rows == self.tp_size * local.shape[0]
) or is_dp_gatherv_active():
dp_reduce_scatter_tensor(local, values)
if _is_hip and values.is_cuda:
dp_reduce_scatter_tensor(
local.view(torch.int32), values.view(torch.int32)
)
else:
dp_reduce_scatter_tensor(local, values)
else:
dp_scatter(local, tensor_model_parallel_all_reduce(values), forward_batch)
dp_scatter(local, self._reduce_owned_rows(values), forward_batch)
return local.view(*indices.shape, self.dim)
+49
View File
@@ -233,6 +233,8 @@ if _use_aiter:
from aiter.fused_moe import fused_topk as aiter_fused_topk
except ImportError:
raise ImportError("aiter is required when SGLANG_USE_AITER is set to True")
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_reduce_partials
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_gate
if _is_musa:
try:
from mate import moe_fused_gate
@@ -641,7 +643,10 @@ class TopK(BaseFusedOp):
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
router_logits_partials: Optional[torch.Tensor] = None,
) -> TopKOutput:
if router_logits_partials is not None:
rocm_router_reduce_partials(router_logits_partials, router_logits)
self.topk_config.torch_native = True
topk_output = select_experts(
hidden_states=hidden_states,
@@ -662,7 +667,11 @@ class TopK(BaseFusedOp):
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
router_logits_partials: Optional[torch.Tensor] = None,
) -> TopKOutput:
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
fixed-order sum is the logits; router_logits is then a buffer the fused gate
fills, and any other reader first reduces the partials into it."""
if dynamic_expert_bias is not None:
output_format = TopKOutputFormat.STANDARD
elif self.topk_config.output_format is not None:
@@ -688,6 +697,13 @@ class TopK(BaseFusedOp):
else:
output_format = TopKOutputFormat.STANDARD
if (
router_logits_partials is not None
and output_format != TopKOutputFormat.STANDARD
):
rocm_router_reduce_partials(router_logits_partials, router_logits)
router_logits_partials = None
if output_format == TopKOutputFormat.TRITON_KERNEL:
# renormalize=True is equivalent to sm_first=False
(
@@ -729,6 +745,7 @@ class TopK(BaseFusedOp):
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
dynamic_expert_bias=dynamic_expert_bias,
router_logits_partials=router_logits_partials,
)
return self._apply_waterfill(topk_output, hidden_states.shape[0])
@@ -1423,11 +1440,23 @@ def biased_topk_jit_kernel_impl(
apply_routed_scaling_factor_on_output: Optional[bool] = False,
packed_out: Optional[torch.Tensor] = None,
sqrtsoftplus_log1p: bool = False,
router_logits_partials: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0:
assert packed_out is None, "aiter topk_gating cannot emit packed ids"
if router_logits_partials is not None:
# ROCm decode router: split-K reduce + gate in one launch
return rocm_router_gate(
gating_output,
correction_bias,
topk,
renormalize,
routed_scaling_factor,
partials=router_logits_partials,
)
from aiter import topk_gating
num_tokens = gating_output.shape[0]
@@ -1451,6 +1480,7 @@ def biased_topk_jit_kernel_impl(
return topk_weights, topk_ids
else:
assert router_logits_partials is None
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
# DeepSeek-V4 stores e_score_correction_bias in bf16 (for the aiter
@@ -2460,6 +2490,7 @@ def select_experts(
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
router_logits_partials: Optional[torch.Tensor] = None,
) -> StandardTopKOutput:
top_k = topk_config.top_k
use_grouped_topk = topk_config.use_grouped_topk
@@ -2495,6 +2526,22 @@ def select_experts(
)
routing_overridden = simulate_uniform_experts or simulate_round_robin_experts
if router_logits_partials is not None and not (
_use_aiter
and scoring_func == "sqrtsoftplus"
and custom_routing_function is None
and not use_grouped_topk
and not torch_native
and expert_location_dispatch_info is None
and (
num_fused_shared_experts == 0
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
)
):
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
rocm_router_reduce_partials(router_logits_partials, router_logits)
router_logits_partials = None
(
router_logits,
correction_bias,
@@ -2619,6 +2666,8 @@ def select_experts(
_packed_kwargs = dict(packed_out=packed_topk)
if topk_config.sqrtsoftplus_log1p:
_packed_kwargs["sqrtsoftplus_log1p"] = True
if router_logits_partials is not None:
_packed_kwargs["router_logits_partials"] = router_logits_partials
topk_weights, topk_ids = _biased_topk(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -126,6 +126,9 @@ _is_cpu_amx_available = cpu_has_amx_support()
_is_cpu = is_cpu()
_is_fp8_fnuz = is_fp8_fnuz()
_is_gfx95_supported = is_gfx95_supported()
if _is_hip:
from sglang.srt.layers.quantization import fp8_hip
# gfx942 (MI300) has no MX matmul HW; MXFP8 checkpoints are converted to
# block-fp8 [128,128] at load and run through the native block-fp8 kernels.
# SGLANG_FORCE_MXFP8_BLOCK_CONVERT=1 opts into that same block-fp8 path on
@@ -951,6 +954,8 @@ class Fp8LinearMethod(LinearMethodBase):
"weight_scale_inv_swizzled",
block_scale_interleave(scale_u8.contiguous()).contiguous(),
)
elif backend.is_gfx95():
fp8_hip.process_dense_weights(self, layer, scale_u8)
elif backend.is_deep_gemm():
from sglang.srt.layers.deep_gemm_wrapper.configurer import (
DEEPGEMM_SCALE_UE8M0,
@@ -1162,6 +1167,9 @@ class Fp8LinearMethod(LinearMethodBase):
bias=bias,
)
if _is_hip and self.block_fp8_as_mxfp8 and self.mxfp8_dense_backend.is_gfx95():
return fp8_hip.apply_dense(self, layer, x, bias)
mxfp8_view = self.use_mxfp8 or (
self.block_fp8_as_mxfp8 and layer.block_fp8_mxfp8_ready
)
@@ -0,0 +1,126 @@
"""gfx950 dense routes of Fp8LinearMethod for 32x32-block fp8 checkpoints served as
MXFP8 (block_fp8_as_mxfp8): the Triton dot_scaled kernel, or the native scaled-MFMA
kernels on a lane-ordered weight. The fused gfx950 producers hand these routes their
operand as an Fp8GridActivation (bf16 already on the fp8 grid) or an Mxfp8Activation."""
from __future__ import annotations
from typing import Optional
import torch
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
Fp8GridActivation,
Mxfp8Activation,
bf16_dequant_blockscaled_linear,
dequant_block_fp8_weight_to_bf16,
dequant_mxfp8_to_bf16,
)
from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import (
native_route_supports,
prepare_mxfp8_native_weight,
)
from sglang.srt.layers.utils import copy_or_rebind_param
def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None:
"""The gfx950 branch of Fp8LinearMethod.process_weights_after_loading."""
backend = method.mxfp8_dense_backend
if backend.is_gfx95_dot_scaled():
# dot_scaled reads canonical [N, K // 32] e8m0 bytes; block scales stay for direct readers
if scale_u8 is not None:
copy_or_rebind_param(layer, "weight_scale_inv_mx", scale_u8.contiguous())
return
assert backend.is_gfx95_mxfp8_native()
n, k = layer.weight.shape
layer.mxfp8_native_ready = False
if native_route_supports(n, k):
# same bytes in scaled-MFMA lane order
shuffled, scale_ue8m0 = prepare_mxfp8_native_weight(
layer.weight.data,
layer.weight_scale_inv.data,
method.weight_block_size,
)
copy_or_rebind_param(layer, "weight", shuffled.view(torch.float8_e4m3fn))
copy_or_rebind_param(layer, "weight_scale_mx_e8m0", scale_ue8m0)
layer.mxfp8_native_ready = True
else:
# a shape the native kernels do not tile keeps the bf16-dequant route
copy_or_rebind_param(
layer,
"weight_bf16",
dequant_block_fp8_weight_to_bf16(
layer.weight.data,
layer.weight_scale_inv.data,
method.weight_block_size,
),
)
def apply_dense(
method, layer: torch.nn.Module, x, bias: Optional[torch.Tensor]
) -> torch.Tensor:
"""Fp8LinearMethod.apply on a gfx950 route. x is a bf16 tensor, an (fp8, scale)
tuple from a fused quant kernel, or one of the wrappers the fused producers emit."""
backend = method.mxfp8_dense_backend
mxfp8_ready = layer.block_fp8_mxfp8_ready
native_route = mxfp8_ready and backend.is_gfx95_mxfp8_native()
# Unwrap the producer's operand: the native route takes fp8 + scales or the fp8-grid
# bf16 directly; the dot_scaled route quantizes the plain tensor itself (per-32
# rounding is idempotent, so the wrapper's rounding is exact for it).
input_scale, on_fp8_grid = None, False
if isinstance(x, Mxfp8Activation):
if native_route:
x, input_scale = x.q, x.scale
else:
x = dequant_mxfp8_to_bf16(x.q, x.scale)
elif isinstance(x, Fp8GridActivation):
x, on_fp8_grid = x.x, native_route
elif isinstance(x, tuple):
x, input_scale = x
if native_route:
return _apply_native(method, layer, x, bias, input_scale, on_fp8_grid)
if mxfp8_ready and input_scale is None:
return method.w8a8_mxfp8_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale_inv_mx,
input_scale=None,
bias=bias,
)
# a shape the gfx950 kernels do not tile, or a pre-quantized tuple off the native route
return method.w8a8_block_fp8_linear(
input=x,
weight=layer.weight,
block_size=method.weight_block_size,
weight_scale=layer.weight_scale_inv,
input_scale=input_scale,
bias=bias,
)
def _apply_native(
method,
layer: torch.nn.Module,
x: torch.Tensor,
bias: Optional[torch.Tensor],
input_scale: Optional[torch.Tensor] = None,
input_on_fp8_grid: bool = False,
) -> torch.Tensor:
"""The native route (mxfp8_native_amd_gfx95); a layer whose shape the native
kernels do not tile keeps the bf16-dequant route."""
if layer.mxfp8_native_ready:
# an fp8-grid bf16 input re-encodes exactly, so it needs no flag here
return method.w8a8_mxfp8_linear(
input=x,
weight_shuffled=layer.weight.view(torch.uint8),
weight_scale_ue8m0=layer.weight_scale_mx_e8m0,
input_scale=input_scale,
bias=bias,
)
if input_scale is not None:
x = dequant_mxfp8_to_bf16(x, input_scale)
input_on_fp8_grid = True
return bf16_dequant_blockscaled_linear(
x, layer.weight_bf16, bias=bias, input_on_fp8_grid=input_on_fp8_grid
)
@@ -381,6 +381,8 @@ class Mxfp8DenseGemmBackend(Enum):
FLASHINFER_TRTLLM = "flashinfer_trtllm"
DEEP_GEMM = "deep_gemm"
GFX95_DOT_SCALED = "gfx95_dot_scaled"
# gfx950 native MXFP8: lane-ordered fp8 weight + ue8m0 scale bytes; untiled shapes keep bf16
GFX95_MXFP8_NATIVE = "gfx95_mxfp8_native"
UNSUPPORTED = "unsupported"
def is_flashinfer_cutlass(self) -> bool:
@@ -401,6 +403,15 @@ class Mxfp8DenseGemmBackend(Enum):
def is_gfx95_dot_scaled(self) -> bool:
return self == Mxfp8DenseGemmBackend.GFX95_DOT_SCALED
def is_gfx95_mxfp8_native(self) -> bool:
return self == Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE
def is_gfx95(self) -> bool:
return self in (
Mxfp8DenseGemmBackend.GFX95_DOT_SCALED,
Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE,
)
def is_unsupported(self) -> bool:
return self == Mxfp8DenseGemmBackend.UNSUPPORTED
@@ -724,8 +735,14 @@ def _unsupported_mxfp8_linear(*args, **kwargs) -> torch.Tensor:
def resolve_block_fp8_mxfp8_backend() -> Mxfp8DenseGemmBackend:
"""The FlashInfer MXFP8 backend a 32-wide-K ue8m0 block-fp8 weight can run on."""
"""The MXFP8 backend (FlashInfer on CUDA, gfx950 on ROCm) a 32-wide-K ue8m0
block-fp8 weight can run on."""
backend = get_fp8_gemm_runner_backend()
# the Triton block kernel's ue8m0 activation quant is CUDA-only, so gfx950 takes an MXFP8 route
if _is_hip and _is_gfx95_supported:
if backend.is_triton():
return Mxfp8DenseGemmBackend.GFX95_DOT_SCALED
return Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE
# Explicit CUTLASS / CuTe-DSL only: they leave the weight untouched and store
# the swizzled scale separately, so the block layout stays readable by Triton.
if not (backend.is_flashinfer_cutedsl() or backend.is_flashinfer_cutlass()):
@@ -758,6 +775,18 @@ def dispatch_block_fp8_mxfp8_linear(backend: Mxfp8DenseGemmBackend) -> Callable:
return partial(
flashinfer_mxfp8_blockscaled_linear, backend="cute-dsl", pin_tactic=True
)
if backend.is_gfx95_dot_scaled():
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
dot_scaled_mxfp8_blockscaled_linear,
)
return dot_scaled_mxfp8_blockscaled_linear
if backend.is_gfx95_mxfp8_native():
from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import (
mxfp8_native_blockscaled_linear,
)
return mxfp8_native_blockscaled_linear
return _unsupported_mxfp8_linear
@@ -1,8 +1,12 @@
from typing import Optional, Tuple
import torch
from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla
from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat
from aiter.tuned_gemm import tgemm
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_gfx95_supported
__all__ = [
@@ -31,6 +35,27 @@ def aiter_dsv3_router_gemm(
return tgemm.mm(hidden_states, weight.detach(), otype=hidden_states.dtype)
def rocm_dsv3_router_split_k(
gate, hidden_states: torch.Tensor
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""The ROCm decode router for gate (a MoEGate): an fp32 logits buffer plus
the split-K partials whose fixed-order sum fills it, or None when gate.forward
applies. Only TopK.forward_cuda(..., router_logits_partials=partials) may read
the buffer: it sums the partials into it inside the fused gate launch."""
num_tokens = hidden_states.shape[0]
if not 0 < num_tokens <= gate.rocm_router_max_tokens:
return None
if get_exec().deterministic.enable_deterministic_inference:
return None
partials = rocm_router_gemv_split_k(hidden_states, gate.weight)
logits = torch.empty(
(num_tokens, gate.weight.shape[0]),
dtype=torch.float32,
device=hidden_states.device,
)
return logits, partials
def get_dsv3_gemm_output_zero_allocator_size(
n_routed_experts: int, num_moe_layers: int, allocate_size: int, embedding_dim: int
):
@@ -23,11 +23,12 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import (
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backend
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool
from sglang.srt.mem_cache.memory_pool import KVCache
from sglang.srt.runtime_context import get_exec, get_platform, get_spec
from sglang.srt.utils import ceil_div, is_hip
from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip
logger = logging.getLogger(__name__)
@@ -170,7 +171,8 @@ def flashmla_supports_v41_kv_layouts() -> bool:
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
"""The (main-cache layout, compressed-cache option) for a new DeepSeek-V4
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA."""
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950
AITER attention."""
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
if mode == "v4":
@@ -181,6 +183,17 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
and torch.version.cuda is not None
and torch.cuda.get_device_capability()[0] == 10
)
if _is_hip:
supported = (
is_gfx95_supported() and resolve_hip_flashmla_backend() == "aiter_sparse"
)
if mode == "auto" and not supported:
return KVLayout.V4, None
if not supported:
raise ValueError(
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
)
return KVLayout.V41, option
supported = flashmla_supports_v41_kv_layouts()
if mode == "auto":
if is_sm100 and supported:
@@ -665,6 +678,21 @@ class DeepSeekV4IndexerPool(KVCache):
loc: torch.Tensor,
cache_k: torch.Tensor,
) -> None:
if self.uses_aiter_fp4_layout:
# the FlyDSL kernels read the fp4 payload and the packed ue8m0 scales
# from two buffers
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
store_fp4_index_k_cache_split,
)
return store_fp4_index_k_cache_split(
cache_k,
self.index_k_payload_buffer[layer_id - self.start_layer],
self.index_k_scale_buffer[layer_id - self.start_layer],
loc,
page_size=self.page_size,
rne=self.index_k_rne,
)
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
store_fp4_index_k_cache,
)
@@ -1894,6 +1922,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
compress_layer_id
)
def low_ratio_index_k_is_split(self, layer_id: int) -> bool:
"""Whether the layer's index-K pool keeps payload and scale in the split
FlyDSL layout (ROCm) rather than one fused [.., 68]-byte row."""
compress_ratio, _, _ = self.layer_mapping[layer_id]
return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout
def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor:
self.wait_layer_transfer(layer_id)
compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id]
@@ -1990,8 +2024,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
eps: float,
freqs_cis: torch.Tensor,
positions: torch.Tensor,
q: Optional[torch.Tensor] = None,
) -> None:
"""q ([B, H, head_dim]): rope its query heads in the same launch."""
if self.uniform_fp8:
assert q is None, "uniform FP8 store does not fuse query RoPE"
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
# then an e4m3 cast + scatter with per-tensor scale 1.0.
from sglang.kernels.ops.attention.deepseek_v4_rope import (
@@ -2019,6 +2056,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
kvcache=self.get_swa_raw_buffer(layer_id),
page_size=self.swa_page_size,
layout=self.kv_layout,
q=q,
)
def set_unified_key_buffer_radix_fused_norm_rope(
@@ -0,0 +1,53 @@
"""ROCm activation route of DeepseekV2MLP: aiter's fused clamp + silu-and-mul for 128-wide half
widths (fp8 out for a 128x128-block down_proj), else the Triton silu-and-mul-clamp, whose gfx950
epilogue lands on down_proj's fp8 grid or emits native MXFP8. Bound by deepseek_v2 under _is_hip."""
from __future__ import annotations
import torch
from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import (
silu_and_mul_clamp_fp8_grid_supported,
silu_and_mul_clamp_triton,
)
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
def resolve_fused_clamp_route(mlp, half_width: int) -> None:
"""Fix mlp's activation route from down_proj's loaded weight (once per layer)."""
quant_method = mlp.down_proj.quant_method
# the aiter kernel tiles and quantizes the half width per 128, the 128x128 block GEMM's layout
mlp.use_fused_clamp_act_mul = half_width % 128 == 0
mlp._fused_clamp_use_fp8 = (
isinstance(quant_method, Fp8LinearMethod)
and quant_method.block_quant
and quant_method.weight_block_size == [128, 128]
)
# gfx950 32-block route: the activation lands on the fp8 grid, so down_proj skips its fake-quant
mlp._hip_act_fp8_grid = bool(
isinstance(quant_method, Fp8LinearMethod)
and quant_method.block_fp8_as_mxfp8
and mlp.down_proj.block_fp8_mxfp8_ready
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
and silu_and_mul_clamp_fp8_grid_supported(half_width)
)
# the native MXFP8 route takes fp8 + ue8m0 straight from the epilogue at decode token counts
mlp._hip_act_native_consumer = bool(
mlp._hip_act_fp8_grid
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
and mlp.down_proj.mxfp8_native_ready
)
mlp._fused_clamp_fp8_checked = True
def silu_and_mul_clamp(mlp, gate_up: torch.Tensor):
"""silu(clamp(g)) * clamp(u) for any half width (unlike the aiter kernel's
multiple of 128), emitted on down_proj's fp8 grid or as native fp8 + ue8m0
when the resolved route takes it."""
return silu_and_mul_clamp_triton(
gate_up,
float(mlp.swiglu_limit),
fp8_grid=mlp._hip_act_fp8_grid,
# the native down_proj runs on fp8 + ue8m0 at every M, so hand it those directly
emit_fp8=mlp._hip_act_native_consumer,
)
@@ -0,0 +1,84 @@
"""ROCm glue of DeepseekV2MoE: V4.1's split-K decode router and the fused all-reduce + mHC
post of the DeepSeek-V4 decode batches."""
from __future__ import annotations
from typing import Optional, Tuple
import torch
from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
ALL_REDUCE_MHC_MAX_ROWS,
)
from sglang.srt.models.deepseek_common.utils import _use_aiter
from sglang.srt.runtime_context import get_parallel
if _use_aiter:
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens
from sglang.srt.layers.rocm_linear_utils import rocm_dsv3_router_split_k
def router_max_tokens(gate, config, is_hash_moe: bool) -> int:
"""Rows up to which the split-K router serves gate (a MoEGate), -1 when it never does:
V4.1 only, DSv4 keeps aiter's router GEMM and gate."""
if not (
_use_aiter
and gate.is_deepseek_v4
and getattr(config, "model_type", None) == "deepseek_v41"
and not is_hash_moe
):
return -1
return rocm_router_max_tokens(
num_experts=config.n_routed_experts,
hidden_size=config.hidden_size,
topk=config.num_experts_per_tok,
weight_dtype=gate.weight.dtype,
)
def batch_has_images(forward_batch) -> bool:
"""Whether a batch can carry image tokens: only extend batches with image inputs do, so
every other batch takes the fused top-k instead of the vision one."""
if forward_batch is None:
return True
return (
forward_batch.forward_mode.is_extend() and forward_batch.contains_image_inputs()
)
def forward_gate(
moe, hidden_states: torch.Tensor, gemm_output_zero_allocator, *, fused_gate: bool
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
"""The router logits of moe (a DeepseekV2MoE) and, on the split-K decode router, the
partials moe.topk sums into them; fused_gate=False when anything else reads them."""
if fused_gate and _use_aiter and not moe.is_hash:
logits_and_partials = rocm_dsv3_router_split_k(moe.gate, hidden_states)
if logits_and_partials is not None:
return logits_and_partials
return moe.gate(hidden_states, gemm_output_zero_allocator), None
def fused_all_reduce_mhc(
moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor
) -> bool:
"""Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the
mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output
and hidden_states only keeps the DP wrapper's tensor contract."""
if not (
mhc is not None
and not moe._shared_expert_tp1
and not mhc.overlap_only
and mhc.post is not None
and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
):
return False
mhc.output = all_reduce_mhc_post(
hidden_states,
mhc.residual,
mhc.post,
mhc.comb,
get_parallel().tp_group.ca_comm,
)
return True
@@ -1,11 +1,22 @@
import logging
from typing import Optional, Tuple
from typing import TYPE_CHECKING, Optional, Tuple
import torch
import triton
if TYPE_CHECKING:
from sglang.kernels.ops.layernorm.mhc_boundary_hip import HcCoefficients
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_platform
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion
from sglang.srt.runtime_context import (
get_exec,
get_forward,
get_parallel,
get_platform,
)
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip
from sglang.srt.utils.common import is_gfx1250_supported
@@ -15,6 +26,22 @@ _is_hip = is_hip()
_is_gfx95_supported = is_gfx95_supported()
_is_gfx1250_supported = is_gfx1250_supported()
if _is_hip:
from sglang.kernels.ops.communication.all_reduce_mhc_hip import (
all_reduce_mhc_post,
)
from sglang.kernels.ops.layernorm.mhc_boundary_hip import (
hc_boundary_fused_deferred,
)
if _is_gfx95_supported:
from sglang.srt.models.deepseek_common.amd.deepseek_v4_gfx95_dense import (
post_attention_norm,
)
# the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows
MHC_HIDDEN_SIZE = 5120
ALL_REDUCE_MHC_MAX_ROWS = 8
_FUSED_HC_POST_PRE_M_THRESHOLD = 64
_FUSED_HC_POST_PRE_CACHE: dict[tuple, dict[str, torch.Tensor]] = {}
_TRITON_MHC_POST_PRE_OPS = None
@@ -421,3 +448,178 @@ def apply_mhc_post_pre_boundary(
)
post_out = post_out.squeeze(-1) if post_out.ndim == 3 else post_out
return residual, layer_input_out, post_out, comb_out, True
def hc_boundary(
layer,
x: Optional[torch.Tensor],
residual: torch.Tensor,
post: Optional[torch.Tensor],
comb: Optional[torch.Tensor],
pre_prev: Optional[torch.Tensor],
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor, "HcCoefficients"]:
"""Fused sublayer boundary (ROCm): apply the pending hc_post of x onto residual, collapse
with pre_prev (copy 0 when None) and take the mixing statistics. Returns (new_residual, y,
coefficients); the coefficients' reduce + sinkhorn is still pending (see HcCoefficients)."""
new_residual, y, coefficients = hc_boundary_fused_deferred(
x,
residual,
post,
comb,
pre_prev,
hc_fn,
hc_scale,
hc_base,
layer.hc_mult,
layer.hc_sinkhorn_iters,
layer.rms_norm_eps,
layer.hc_eps,
)
if new_residual is None:
new_residual = residual
if y is None:
y = new_residual[:, 0, :].contiguous()
return new_residual, y, coefficients
def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool:
return (
_is_gfx95_supported
and 1 <= residual.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
and residual.shape[1:] == (4, MHC_HIDDEN_SIZE)
and residual.dtype == torch.bfloat16
and residual.is_contiguous()
and layer.config.model_type == "deepseek_v41"
and (
forward_batch.forward_mode.is_decode()
or forward_batch.forward_mode.is_target_verify()
)
and get_parallel().attn_dp_size == 1
and get_parallel().tp_size == 4
and not layer.dsa_enable_prefill_cp
and not get_forward().sp_active
and not is_batch_invariant_mode_enabled()
and not get_exec().deterministic.enable_deterministic_inference
)
def _make_mhc_fusion(
residual: torch.Tensor, coefficients: "HcCoefficients", comm
) -> Optional[MhcPostFusion]:
# comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off
if comm is None or comm.disabled or not comm.enable_register_for_capturing:
return None
coefficients.materialize()
return MhcPostFusion(
residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre
)
def attention_mhc_fusion(layer, residual, coefficients, forward_batch):
if not (
_can_fuse_mhc(layer, residual, forward_batch)
and layer.self_attn.attn_tp_size == 4
and layer.self_attn.wo_b.reduce_results
):
return None
return _make_mhc_fusion(
residual, coefficients, get_parallel().attn_tp_group.ca_comm
)
def moe_mhc_fusion(layer, residual, coefficients, forward_batch):
if not (
_can_fuse_mhc(layer, residual, forward_batch)
and layer.mlp.tp_size == 4
and not layer.mlp._shared_expert_tp1
and get_moe_a2a_backend().is_none()
):
return None
return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm)
def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None:
# a lazily recorded state still needs its stats before the fused kernel reads them
state.materialize_stats()
if state.stats_stream is not None:
torch.cuda.current_stream().wait_stream(state.stats_stream)
state.output = all_reduce_mhc_post(
x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm
)
def forward_hc_pre_from_prev_fused_boundary(
layer,
positions: torch.Tensor,
hidden_states: Optional[torch.Tensor],
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
prev_pre: Optional[torch.Tensor],
pending_post: Optional[Tuple[torch.Tensor, ...]],
defer_post: bool,
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
"""ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev.
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
if pending_post is None:
pending_post = (None, hidden_states, None, None)
residual, x, attn_coefficients = hc_boundary(
layer,
*pending_post,
prev_pre,
layer.hc_attn_fn,
layer.hc_attn_scale,
layer.hc_attn_base,
)
# the boundary's reduce + sinkhorn rides in the norm launch
x, x_quant = layer._input_norm(
x, allow_aiter_quant=False, coefficients=attn_coefficients
)
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
with use_mhc_post_fusion(mhc):
x = layer.self_attn(
x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant
)
if mhc is not None:
residual = mhc.output
x = None
residual, x, ffn_coefficients = hc_boundary(
layer,
x,
residual,
attn_coefficients.post if mhc is None else None,
attn_coefficients.comb if mhc is None else None,
attn_coefficients.pre,
layer.hc_ffn_fn,
layer.hc_ffn_scale,
layer.hc_ffn_base,
)
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
with use_mhc_post_fusion(mhc):
x = layer._run_moe_ffn_dp_sync(
x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global
)
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
if mhc is not None and mhc.output is not None:
# Reduction already applied post. The next boundary consumes this
# materialized residual, including when it would normally defer post.
return mhc.output, ffn_pre, None
if defer_post:
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients):
"""layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where
the gfx950 norm kernel is available; the module norm plus a standalone reduce +
sinkhorn elsewhere."""
if _is_gfx95_supported:
return post_attention_norm(layer, x, coefficients)
coefficients.materialize()
return layer.post_attention_layernorm(x)
@@ -0,0 +1,205 @@
"""gfx950 dense route of the DeepSeek-V4 attention for 32-wide-block (V4.1) fp8 checkpoints: the
fused RMSNorm + fake-quant producers hand wqkv_a / wq_b their operand on the fp8 grid
(Fp8GridActivation) or as native MXFP8 (Mxfp8Activation); deepseek_v4 binds it under _is_hip."""
from __future__ import annotations
import logging
from typing import Optional, Tuple
import torch
from torch import nn
from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
Fp8GridActivation,
Mxfp8Activation,
)
from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import (
rmsnorm_fake_quant_fp8,
)
from sglang.srt.environ import envs
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from sglang.srt.layers.quantization.fp8_utils import resolve_block_fp8_mxfp8_backend
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_gfx95_supported, is_hip
from sglang.srt.utils.common import is_gfx1250_supported
logger = logging.getLogger(__name__)
_is_hip = is_hip()
_is_gfx95_supported = is_gfx95_supported()
_is_gfx1250_supported = is_gfx1250_supported()
_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip
# V4.1's wo_a route on gfx950: an aiter batched GEMM fork with wo_b's fp8-grid rounding in its
# epilogue; None keeps the aiter kernel
_wo_a_fp8_grid_gemm = None
if _use_aiter and _is_gfx95_supported:
from sglang.kernels.ops.gemm.gfx95_batched_gemm_bf16_fp8_grid import (
batched_gemm_bf16_fp8_grid as _wo_a_fp8_grid_gemm,
)
def fused_rmsnorm_fp8_quant_eligible(
quant_config: Optional[QuantizationConfig],
) -> bool:
"""Whether the aiter fused RMSNorm + fp8 quant applies: its (fp8, 128-group scale)
output is consumed only by the 128x128-block dense GEMM."""
return bool(
_use_aiter
and (_is_gfx95_supported or _is_gfx1250_supported)
and isinstance(quant_config, Fp8Config)
and quant_config.weight_block_size == [128, 128]
)
def fused_rmsnorm_fake_quant_eligible(
quant_config: Optional[QuantizationConfig],
) -> bool:
"""Whether rmsnorm_fake_quant_fp8 applies: gfx950 with a 32-wide-block checkpoint
(V4.1), whose dense route takes the norm output already on the fp8 grid as an
Fp8GridActivation."""
if not (_is_hip and _is_gfx95_supported and isinstance(quant_config, Fp8Config)):
return False
block = quant_config.weight_block_size
return (
block is not None
and block[1] == 32
and quant_config.scale_fmt == "ue8m0"
and resolve_block_fp8_mxfp8_backend().is_gfx95_mxfp8_native()
)
def _native_mxfp8_consumer(linear: Optional[nn.Module]) -> bool:
"""Whether linear runs the gfx950 native MXFP8 route, which consumes fp8 + ue8m0
scales directly at every M."""
if linear is None:
return False
quant_method = linear.quant_method
return (
isinstance(quant_method, Fp8LinearMethod)
and quant_method.block_fp8_as_mxfp8
and linear.block_fp8_mxfp8_ready
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
and linear.mxfp8_native_ready
)
def _fake_quant_applies(norm: nn.Module, x: torch.Tensor) -> bool:
return x.dim() == 2 and x.dtype == norm.weight.dtype
def q_norm_fake_quant(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]:
"""attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b
consumes, already on the fp8 grid); the plain norm, twice, when the rows are not
a 2-D bf16 batch."""
if not _fake_quant_applies(attn.q_norm, q_lora):
q_lora = attn.q_norm(q_lora)
return q_lora, q_lora
if not attn._wq_b_native_consumer_checked:
attn._wq_b_native_consumer = _native_mxfp8_consumer(attn.wq_b)
attn._wq_b_native_consumer_checked = True
q_for_wq_b, q_lora = rmsnorm_fake_quant_fp8(
q_lora,
attn.q_norm.weight.data,
attn.q_norm.variance_epsilon,
emit_fp8=attn._wq_b_native_consumer,
)
return q_lora, q_for_wq_b
def input_norm_fake_quant(
layer, hidden_states: torch.Tensor, coefficients=None
) -> Tuple[torch.Tensor, Optional[object]]:
"""layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the fp8-grid operand
of its dense projections, or None for non-2-D / non-bf16 rows). coefficients rides in the norm
launch when the fused kernel runs, else it is materialized here."""
norm = layer.input_layernorm
if not _fake_quant_applies(norm, hidden_states):
if coefficients is not None:
coefficients.materialize()
return norm(hidden_states), None
if not layer._wqkv_a_native_consumer_checked:
# wqkv_a exists only when the q / kv projections are fused
layer._wqkv_a_native_consumer = _native_mxfp8_consumer(
layer.self_attn.wqkv_a if layer.self_attn.fuse_wqa_wkv else None
)
layer._wqkv_a_native_consumer_checked = True
emit_fp8 = layer._wqkv_a_native_consumer
if coefficients is not None and not coefficients.materialized:
x_quant, hidden_states = rmsnorm_with_sinkhorn(
hidden_states,
norm.weight.data,
norm.variance_epsilon,
coefficients,
emit_fp8=emit_fp8,
)
return hidden_states, x_quant
x_quant, hidden_states = rmsnorm_fake_quant_fp8(
hidden_states,
norm.weight.data,
norm.variance_epsilon,
emit_fp8=emit_fp8,
)
return hidden_states, x_quant
def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tensor:
"""layer.post_attention_layernorm(x); with coefficients (HcCoefficients of the
boundary that produced x) still pending, the reduce + sinkhorn rides in the norm
launch, which then is the Triton row norm rather than the aiter one."""
norm = layer.post_attention_layernorm
if (
coefficients is None
or coefficients.materialized
or not _fake_quant_applies(norm, x)
):
if coefficients is not None:
coefficients.materialize()
return norm(x)
_, out = rmsnorm_with_sinkhorn(
x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False
)
return out
def live_rows(activation, num_tokens: int):
"""The first num_tokens rows of a break input; the gfx950 fused q_norm
hands the indexer its Fp8GridActivation wrapper, whose rows live in .x."""
if isinstance(activation, Fp8GridActivation):
return Fp8GridActivation(activation.x[:num_tokens])
if isinstance(activation, Mxfp8Activation):
return Mxfp8Activation(activation.q[:num_tokens], activation.scale[:num_tokens])
return activation[:num_tokens]
def wo_b_takes_fp8_grid(attn) -> bool:
"""Whether attn.wo_b consumes an Fp8GridActivation, which the wo_a GEMM
then emits from its epilogue; resolved on first use, once the weights are loaded."""
if attn._wo_b_fp8_grid_operand is None:
quant_method = attn.wo_b.quant_method
attn._wo_b_fp8_grid_operand = bool(
_wo_a_fp8_grid_gemm is not None
and isinstance(quant_method, Fp8LinearMethod)
and quant_method.block_fp8_as_mxfp8
and attn.wo_b.block_fp8_mxfp8_ready
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
)
return attn._wo_b_fp8_grid_operand
def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool):
"""o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched
GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as
an Fp8GridActivation [T, G * R]; None when the fork did not import."""
if _wo_a_fp8_grid_gemm is None:
return None
# the split-K regime ends at 64 rows, so a request's verify and decode rows would
# take different reduction orders; deterministic inference keeps the single chain
split_k = False if get_exec().deterministic.enable_deterministic_inference else None
y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k)
if fp8_grid:
return Fp8GridActivation(y)
return y.view(o.shape[0], wo_a.shape[0], wo_a.shape[1])
@@ -0,0 +1,247 @@
"""ROCm glue of deepseek_v4: the branches the model takes under _is_hip, bound there as
_hip. The gfx950 dense fp8 routes live in deepseek_v4_gfx95_dense, the fused mHC
boundary in deepseek_v4_fused_mhc."""
from __future__ import annotations
from typing import Optional, Tuple
import torch
from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
from sglang.srt.environ import envs
from sglang.srt.layers.attention.hip_flash_mla import (
hip_attention_needs_head_pad,
resolve_hip_flashmla_backend,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
is_in_breakable_cuda_graph,
)
from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module
apply_attention_mhc,
forward_hc_pre_from_prev_fused_boundary,
)
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_gfx95_supported
live_rows = gfx95_dense.live_rows
wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul
_is_gfx95_supported = is_gfx95_supported()
# ---- MqaAttentionBase / MQALayer ----
def init_mqa_layer(attn, quant_config) -> None:
"""The ROCm state of MQALayer.__init__. On the gfx950 32-block route q_norm also emits
the fp8-grid operand of wq_b; whether wq_b consumes native MXFP8, and wo_b the fp8-grid
operand wo_a emits, resolve on first use, once the weights are loaded."""
attn.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible(
quant_config
)
attn._wq_b_native_consumer_checked = False
attn._wq_b_native_consumer = False
attn._wo_b_fp8_grid_operand = None
def wo_a_emits_fp8_grid(attn) -> bool:
"""Whether V4.1's gfx950 wo_a GEMM rounds its output onto wo_b's fp8 grid."""
return attn.is_dsv41 and gfx95_dense.wo_b_takes_fp8_grid(attn)
def wo_a_split_k_allowed() -> bool:
"""Whether the V4.1 gfx950 wo_a decode / verify kernels may run: their split-K
reduction order depends on the row count, which batch-invariant and deterministic
inference rule out."""
return not (
is_batch_invariant_mode_enabled()
or get_exec().deterministic.enable_deterministic_inference
)
def use_fused_qk_norm_rope(attn) -> bool:
return bool(envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get() and attn.q_head_norm)
def q_norm_for_wq_b(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]:
"""attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b
consumes); on the gfx950 32-block route the second is already on the fp8 grid."""
if attn.fused_rmsnorm_fake_quant:
return gfx95_dense.q_norm_fake_quant(attn, q_lora)
q_lora = attn.q_norm(q_lora)
return q_lora, q_lora
def fuses_q_rope_into_k_store(
attn, q_out: Optional[torch.Tensor], *, unified: bool, use_cp: bool
) -> bool:
"""Whether the K norm-rope-store launch ropes the query heads too: only the plain store
path, where it is the cache writer and the query is consumed in place (no padded copy)."""
return (
not unified
and not use_cp
and q_out is None
and not attn.q_head_norm
and not envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get()
)
def wq_b_unroped(attn, q) -> torch.Tensor:
"""attn._compute_q_b without the RoPE, which the K store launch applies."""
q, _ = attn.wq_b(q)
return q.view(-1, attn.n_local_heads, attn.head_dim)
def compute_q_b(
attn,
q_lora: torch.Tensor,
q_for_wqb,
positions: torch.Tensor,
q_out: Optional[torch.Tensor],
unified: bool,
use_cp: bool,
):
"""attn._compute_q_b(q_for_wqb, positions, q_out) as (q, the q_lora the indexer reads,
whether the K norm-rope-store launch ropes q). The indexer takes the fused operand:
its wq_b would re-round onto the same grid."""
fuse_q_rope = _is_gfx95_supported and fuses_q_rope_into_k_store(
attn, q_out, unified=unified, use_cp=use_cp
)
if fuse_q_rope:
q = wq_b_unroped(attn, q_for_wqb)
else:
q = attn._compute_q_b(q_for_wqb, positions, q_out)
return q, q_for_wqb, fuse_q_rope
def skip_head_pad(attn) -> bool:
# only tilelang is built for padded head widths; aiter and Triton take the real head count
return attn.attn_tp_size > 1 and not hip_attention_needs_head_pad()
def attention_inv_rope(
attn,
positions: torch.Tensor,
forward_batch,
unified: bool,
*,
wo_a_applies_inv_rope: bool,
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
"""(freqs_real, positions) for aiter's gfx950 sparse kernel to apply the inverse RoPE
itself; None for the other kernels, and where the fp8 wo_a front end or the prefill
graph op keeps it."""
if (
unified
or not _is_gfx95_supported
or resolve_hip_flashmla_backend() != "aiter_sparse"
or wo_a_applies_inv_rope
):
return None
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
return None
return torch.view_as_real(attn.freqs_cis).flatten(-2), positions
# ---- DeepseekV4DecoderLayer ----
def init_decoder_layer(layer, quant_config) -> None:
"""The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the
pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8
resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and
the mixing stats; the kernel only supports hc_mult 4."""
layer.fused_rmsnorm_fp8_quant = gfx95_dense.fused_rmsnorm_fp8_quant_eligible(
quant_config
)
layer.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible(
quant_config
)
layer._wqkv_a_native_consumer_checked = False
layer._wqkv_a_native_consumer = False
layer.hc_boundary_fused = (
_is_gfx95_supported and layer.hc_pre_from_prev_sublayer and layer.hc_mult == 4
)
def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]:
"""V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks,
whose ordinary stores preserve locality for the following mHC reader; None elsewhere."""
if not (
_is_gfx95_supported
and layer.config.model_type == "deepseek_v41"
and 768 <= x.shape[0] <= 4096
and x.shape[1] == 5120
and residual.shape == (x.shape[0], 4, 5120)
and post.shape == (x.shape[0], 4)
and comb.shape == (x.shape[0], 4, 4)
and x.dtype == residual.dtype == torch.bfloat16
and post.dtype == comb.dtype == torch.float32
and all(t.is_contiguous() for t in (x, residual, post, comb))
):
return None
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
def input_norm(
layer,
hidden_states: torch.Tensor,
allow_aiter_quant: bool,
coefficients,
fused_rmsnorm_fp8_quant,
) -> Tuple[torch.Tensor, Optional[object]]:
"""layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the
pre-quantized operand of its dense projections or None). coefficients is the fused
boundary's pending reduce + sinkhorn, hosted by the gfx950 norm launch when that one runs."""
if layer.fused_rmsnorm_fp8_quant and allow_aiter_quant:
if coefficients is not None:
coefficients.materialize()
x_quant, hidden_states = fused_rmsnorm_fp8_quant(
hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps
)
return hidden_states, x_quant
if layer.fused_rmsnorm_fake_quant:
return gfx95_dense.input_norm_fake_quant(layer, hidden_states, coefficients)
if coefficients is not None:
coefficients.materialize()
return layer.input_layernorm(hidden_states), None
# ---- DeepseekV4Model layer loop ----
def forward_layer_fused_boundary(
model,
i: int,
*,
positions: torch.Tensor,
hidden_states: Optional[torch.Tensor],
input_ids: torch.Tensor,
forward_batch,
input_ids_global: torch.Tensor,
prev_pre: Optional[torch.Tensor],
pending_post: Optional[Tuple[torch.Tensor, ...]],
capture_dspark: bool,
):
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
DSpark capture) or there is none; hidden_states is None while a post is pending."""
nxt = i + 1
defer_post = (
nxt < model.end_layer
and model.layers[nxt].engram is None
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
)
return forward_hc_pre_from_prev_fused_boundary(
model.layers[i],
positions=positions,
hidden_states=hidden_states,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
prev_pre=prev_pre,
pending_post=pending_post,
defer_post=defer_post,
)
+58 -14
View File
@@ -225,6 +225,13 @@ if _use_aiter_gfx95:
if _use_aiter:
pass
if _is_hip:
from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_act as _hip_act
from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_moe as _hip_moe
else:
_hip_act = None
_hip_moe = None
if _is_cuda:
from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16
elif _is_npu:
@@ -401,20 +408,14 @@ class DeepseekV2MLP(nn.Module):
)
return down_output
if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked:
_hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2)
if self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
from aiter.ops.triton.fusions.fused_clamp_act_mul import (
fused_clamp_act_mul,
)
if not self._fused_clamp_fp8_checked:
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
qm = getattr(self.down_proj, "quant_method", None)
self._fused_clamp_use_fp8 = (
isinstance(qm, Fp8LinearMethod) and qm.block_quant
)
self._fused_clamp_fp8_checked = True
if self._fused_clamp_use_fp8:
from aiter import dtypes
@@ -442,7 +443,9 @@ class DeepseekV2MLP(nn.Module):
# Fallback: fused silu+clamp kernel (still faster than unfused)
elif self.swiglu_limit is not None:
if _is_npu:
if _is_hip:
x = _hip_act.silu_and_mul_clamp(self, gate_up)
elif _is_npu:
x = torch.ops.npu.npu_clipped_swiglu(
gate_up,
alpha=1,
@@ -514,6 +517,10 @@ class MoEGate(nn.Module):
hidden_size=config.hidden_size,
weight_dtype=self.weight.dtype,
)
# Rows up to which the ROCm split-K router serves the gate (-1: never)
self.rocm_router_max_tokens = (
_hip_moe.router_max_tokens(self, config, is_hash_moe) if _is_hip else -1
)
def forward(
self,
@@ -939,6 +946,9 @@ class DeepseekV2MoE(nn.Module):
if forward_batch is not None
else None
)
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
if use_vision_topk and _is_hip:
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
if not self._enable_a2a_moe:
if self._can_dual_stream_graph(hidden_states):
fwd = get_forward()
@@ -961,6 +971,7 @@ class DeepseekV2MoE(nn.Module):
input_ids,
input_ids_global=input_ids_global,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
)
else:
return self.forward_normal(
@@ -970,6 +981,7 @@ class DeepseekV2MoE(nn.Module):
input_ids_global=input_ids_global,
skip_shared_experts=skip_shared_experts,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
)
else:
return self.forward_deepep(
@@ -983,6 +995,7 @@ class DeepseekV2MoE(nn.Module):
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
) -> torch.Tensor:
# Note(kpham-sgl): issue order satisfies 3 constraints:
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
@@ -1016,7 +1029,16 @@ class DeepseekV2MoE(nn.Module):
)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
if _is_hip:
router_logits, router_logits_partials = _hip_moe.forward_gate(
self,
hidden_states,
gemm_output_zero_allocator,
fused_gate=not use_flashinfer_trtllm_bypass and not use_vision_topk,
)
else:
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
router_logits_partials = None
if use_flashinfer_trtllm_bypass:
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
@@ -1029,7 +1051,9 @@ class DeepseekV2MoE(nn.Module):
if getattr(self, "is_hash", False)
else {}
)
if self.gate.e_score_correction_bias_vl is not None:
if router_logits_partials is not None:
topk_kwargs["router_logits_partials"] = router_logits_partials
if use_vision_topk:
topk_output = vision_topk(
self,
router_logits,
@@ -1226,6 +1250,10 @@ class DeepseekV2MoE(nn.Module):
)
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
@@ -1243,6 +1271,7 @@ class DeepseekV2MoE(nn.Module):
input_ids_global: Optional[torch.Tensor] = None,
skip_shared_experts: bool = False,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
) -> torch.Tensor:
if hasattr(self, "shared_experts") and use_intel_amx_backend(
self.shared_experts.gate_up_proj
@@ -1277,13 +1306,24 @@ class DeepseekV2MoE(nn.Module):
pre_quant_input=pre_quant_input,
)
# router_logits: (num_tokens, n_experts)
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
if _is_hip:
router_logits, router_logits_partials = _hip_moe.forward_gate(
self,
hidden_states,
gemm_output_zero_allocator,
fused_gate=not use_vision_topk,
)
else:
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
router_logits_partials = None
topk_kwargs = (
{"input_ids": input_ids_global}
if getattr(self, "is_hash", False)
else {}
)
if self.gate.e_score_correction_bias_vl is not None:
if router_logits_partials is not None:
topk_kwargs["router_logits_partials"] = router_logits_partials
if use_vision_topk:
topk_output = vision_topk(
self,
router_logits,
@@ -1382,6 +1422,10 @@ class DeepseekV2MoE(nn.Module):
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
mhc = current_mhc_post_fusion()
if _is_hip and _hip_moe.fused_all_reduce_mhc(
self, mhc, final_hidden_states
):
return final_hidden_states
if mhc is not None:
mhc.start_stats_before_all_reduce()
final_hidden_states = post_experts_all_reduce(final_hidden_states)
+194 -80
View File
@@ -425,7 +425,9 @@ _wo_a_aiter_batched_gemm_disabled = False
_wo_a_fp8_mxscale = None
_wo_a_fp8_mxscale_fused_invrope = None
_wo_a_weight_scale_to_e8m0 = None
_hip = None
if _is_hip:
from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip
from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import (
apply_wo_a_fp8_mxscale,
apply_wo_a_fp8_mxscale_fused_invrope,
@@ -459,31 +461,48 @@ def _apply_wo_a_bf16_matmul(
fuse_mxfp8_quant: bool = False,
is_prefill: bool = False,
fast_path: bool = False,
fp8_grid: bool = False,
) -> torch.Tensor | Mxfp8SwizzledInput:
# o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated
# on the exact validated TP4 shapes and write token-major output directly.
global _wo_a_aiter_batched_gemm_disabled
if (
fast_path
and _is_cuda
# the gfx950 routes, like the CUDA ones, are V4.1's (fast_path); DSv4 keeps the
# aiter batched GEMM / einsum below
hip_fast_path = fast_path and _is_hip and _is_gfx95_supported
hip_decode_verify = (
hip_fast_path
and (
(
is_decode
and o.shape[0] == 1
and (get_platform().is_blackwell or get_platform().is_sm90)
)
or (
is_target_verify
and 0 < o.shape[0] <= 384
and get_platform().is_blackwell
)
or (
is_prefill
and 4096 <= o.shape[0] <= 65536
and get_platform().is_blackwell
(is_decode and o.shape[0] == 1 and o.is_contiguous())
or (is_target_verify and 2 <= o.shape[0] <= 384)
)
and _hip.wo_a_split_k_allowed()
)
if (
(
fast_path
and _is_cuda
and (
(
is_decode
and o.shape[0] == 1
and (get_platform().is_blackwell or get_platform().is_sm90)
)
or (
is_target_verify
and 0 < o.shape[0] <= 384
and get_platform().is_blackwell
)
or (
is_prefill
and 4096 <= o.shape[0] <= 65536
and get_platform().is_blackwell
)
)
)
and o.shape[1:] == (2, 4096)
or hip_decode_verify
or (hip_fast_path and is_prefill and 4096 <= o.shape[0] <= 65536)
) and (
o.shape[1:] == (2, 4096)
and wo_a.shape == (2, 1024, 4096)
and o.dtype == wo_a.dtype == torch.bfloat16
and o.stride(2) == 1
@@ -507,6 +526,11 @@ def _apply_wo_a_bf16_matmul(
o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1)
)
return result
# the 16-row decode kernels also serve V4.1's target-verify rows
if (is_decode or is_target_verify) and hip_fast_path:
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid)
if y is not None:
return y
if (
is_decode
and _wo_a_aiter_batched_gemm_enabled
@@ -722,7 +746,11 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
get_attn_backend().forward_low_ratio_sources(
layer=layer,
x=x[:real_num_tokens],
q_lora=q_lora[:real_num_tokens],
q_lora=(
_hip.live_rows(q_lora, real_num_tokens)
if _is_hip
else q_lora[:real_num_tokens]
),
positions=positions[:real_num_tokens],
forward_batch=forward_batch,
)
@@ -1102,7 +1130,11 @@ class MQALayer(MqaAttentionBase):
self.register_buffer("sin_cache", sin_cache, persistent=False)
if alt_streams is not None and (
(_is_cuda and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get())
(
# the gfx950 overlap is validated on V4.1 only; DSv4 on ROCm keeps one stream
(_is_cuda or (_is_gfx95_supported and self.is_dsv41))
and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get()
)
or (_is_npu and envs.SGLANG_NPU_USE_MULTI_STREAM.get())
):
self.alt_streams = alt_streams[:3]
@@ -1173,9 +1205,7 @@ class MQALayer(MqaAttentionBase):
prefix=add_prefix("attn_mqa", prefix),
)
self.use_fused_qk_norm_rope = (
_is_hip and envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get()
)
self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self)
# Static eligibility; token count and wo_b's output format are checked
# in forward, after weights have loaded.
self.use_fused_wo_a = (
@@ -1188,6 +1218,8 @@ class MQALayer(MqaAttentionBase):
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
)
if _is_hip:
_hip.init_mqa_layer(self, quant_config)
# KV cache write is always fused into the K kernel
# (`_compute_kv_to_cache`), so the legacy "overlap store cache" flag
@@ -1256,6 +1288,8 @@ class MQALayer(MqaAttentionBase):
self, q: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor | Mxfp8SwizzledInput]:
# The indexer needs the BF16 normalized row; wq_b needs the quantized one.
if _is_hip:
return _hip.q_norm_for_wq_b(self, q)
method = self.wq_b.quant_method
if (
_is_cuda
@@ -1310,13 +1344,13 @@ class MQALayer(MqaAttentionBase):
q = q.view(-1, self.n_local_heads, self.head_dim)
if not self.q_head_norm:
if (
_is_cuda
(_is_cuda or _is_gfx95_supported)
and q_out is not None
and (
0 < q.shape[0] <= 8
or (
self.is_dsv41
and get_platform().is_blackwell
and (get_platform().is_blackwell or _is_gfx95_supported)
and self.n_local_heads == 16
and 4096 <= q.shape[0] <= 65536
)
@@ -1354,13 +1388,16 @@ class MQALayer(MqaAttentionBase):
forward_batch: ForwardBatch,
attn_backend,
qkv_a: Optional[torch.Tensor] = None,
q_rope: Optional[torch.Tensor] = None,
) -> None:
"""Fused: rmsnorm + RoPE + write directly to FlashMLA paged cache.
Replaces the bf16-kv-intermediate path. Used everywhere except the DSA
prefill-CP case (which needs bf16 kv for the cross-rank all-gather).
q_rope ([T, H, head_dim]) has its query heads roped by the same launch.
"""
if envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get():
assert q_rope is None
# Quantize the nope payload from bf16-rounded values (the fused
# kernel quantizes from fp32 registers; the bf16 rounding moves
# values across fp8 bins relative to bf16-sourced consumers).
@@ -1384,6 +1421,8 @@ class MQALayer(MqaAttentionBase):
eps=self.eps,
freqs_cis=self.freqs_cis,
positions=positions,
# the query RoPE fused into this store is HIP only
**({} if q_rope is None else {"q": q_rope}),
)
def _compute_kv_bf16(
@@ -1724,8 +1763,8 @@ class MQALayer(MqaAttentionBase):
dtype=x.dtype,
)
else:
q_lora = self.q_norm(q_lora)
q = self._compute_q_b(q_lora, positions, q_out)
q_lora, q_for_wqb = self._normalize_q_lora(q_lora)
q = self._compute_q_b(q_for_wqb, positions, q_out)
self._compute_kv_to_cache(
x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a
)
@@ -1982,7 +2021,13 @@ class MQALayer(MqaAttentionBase):
q_out.copy_(q)
else:
q_lora, q_for_wqb = self._normalize_q_lora(q_lora)
q = self._compute_q_b(q_for_wqb, positions, q_out)
if _is_hip:
q, q_lora, fuse_q_rope = _hip.compute_q_b(
self, q_lora, q_for_wqb, positions, q_out, unified, use_cp
)
else:
q = self._compute_q_b(q_for_wqb, positions, q_out)
fuse_q_rope = False
if unified:
# unified_kv prefill: keep bf16 kv; the backend writes
# the ring AFTER attention (2-source path).
@@ -2028,7 +2073,12 @@ class MQALayer(MqaAttentionBase):
kv = None
else:
self._compute_kv_to_cache(
x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a
x_linear,
positions,
forward_batch,
attn_backend,
qkv_a=qkv_a,
q_rope=q if fuse_q_rope else None,
)
kv = None
@@ -2132,8 +2182,7 @@ class MQALayer(MqaAttentionBase):
)
low_ratio_multi_stream = (
_is_cuda
and get_platform().is_blackwell
((_is_cuda and get_platform().is_blackwell) or _is_gfx95_supported)
and self.compress_ratio in (1, 2)
and self.alt_streams is not None
and (
@@ -2141,8 +2190,11 @@ class MQALayer(MqaAttentionBase):
or (
forward_batch.forward_mode.is_target_verify()
# Other MXFP8 backends may share mutable GEMM workspace.
and getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None)
== Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL
and (
_is_gfx95_supported
or getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None)
== Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL
)
)
)
)
@@ -2223,7 +2275,11 @@ class MQALayer(MqaAttentionBase):
k_rope = rope_pool.new_empty((x.shape[0], rope_pool.shape[-1]))
kernel_num_heads = self.n_local_heads
else:
kernel_num_heads = self._kernel_num_heads(x.shape[0])
kernel_num_heads = (
self.n_local_heads
if _is_hip and not unified and _hip.skip_head_pad(self)
else self._kernel_num_heads(x.shape[0])
)
if kernel_num_heads != self.n_local_heads:
# Backends without an exact-head specialization retain the existing
# padded shape. attn_sink is sliced to this rank and padded to match.
@@ -2318,8 +2374,21 @@ class MQALayer(MqaAttentionBase):
# its normal causally-indexed store from attn_k = kv.
attn_k = kv if kv is not None else q
# HIP: the attention kernel may apply the inverse RoPE itself; the else-branch below keeps it
inv_rope = (
_hip.attention_inv_rope(
self,
positions,
forward_batch,
unified,
wo_a_applies_inv_rope=self.wo_a_fp8
and _wo_a_fp8_mxscale_fused_invrope is not None,
)
if _is_hip
else None
)
if unified:
# only the HIP radix backend takes these two; passing them always would
# only the HIP radix backend takes these; passing them always would
# leave non-ROCm depending on the **_ in its forward() to drop them, and
# no test on that side would notice if the **_ went away
rope_kwargs = {}
@@ -2327,6 +2396,8 @@ class MQALayer(MqaAttentionBase):
rope_kwargs["q_rope"] = q_rope
if k_rope is not None:
rope_kwargs["k_rope"] = k_rope
if inv_rope is not None:
rope_kwargs["inv_rope"] = inv_rope
o = attn_backend.forward(
q=q_out if q_out is not None else q,
k=attn_k,
@@ -2364,6 +2435,8 @@ class MQALayer(MqaAttentionBase):
compress_ratio=self.compress_ratio,
attn_sink=attn_sink,
save_kv_cache=save_kv_cache,
# HIP only, for the same reason as rope_kwargs above
**({} if inv_rope is None else {"inv_rope": inv_rope}),
)
o = o[:, tp_slice, :]
if (
@@ -2405,7 +2478,7 @@ class MQALayer(MqaAttentionBase):
sin4,
qk_nope_dim=self.qk_nope_head_dim,
)
elif not fuse_rope_wo_a:
elif inv_rope is None and not fuse_rope_wo_a:
# The fused path folds this in; it must not run twice.
fused_rope_inplace(
o[..., -self.qk_rope_head_dim :],
@@ -2510,6 +2583,7 @@ class MQALayer(MqaAttentionBase):
is_prefill=forward_batch.forward_mode.is_extend_without_speculative(),
fast_path=self.is_dsv41,
fuse_mxfp8_quant=fuse_mxfp8_quant,
fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self),
)
else:
o = _apply_gguf_grouped_wo_a(
@@ -2519,16 +2593,21 @@ class MQALayer(MqaAttentionBase):
self.o_lora_rank,
)
return self._project_wo_b(o)
def _project_wo_b(self, o):
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
mhc = current_mhc_post_fusion()
o, _ = self.wo_b(
o if isinstance(o, Mxfp8SwizzledInput) else o.flatten(1),
o.flatten(1) if isinstance(o, torch.Tensor) else o,
skip_all_reduce=mhc is not None,
)
if mhc is not None and mhc.overlap_only:
mhc.start_stats_before_all_reduce()
o = attn_tp_all_reduce(o)
elif mhc is not None and _is_hip:
_hip.apply_attention_mhc(o, mhc)
elif mhc is not None:
from sglang.kernels.ops.communication.all_reduce_mhc import (
all_reduce_mhc_norm,
@@ -2679,6 +2758,8 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_pre_from_prev_sublayer = config.hc_pre_from_prev_sublayer
if self.hc_pre_from_prev_sublayer:
self.use_fused_mhc_post_pre = False
if _is_hip:
_hip.init_decoder_layer(self, quant_config)
self.engram = None
if engram_layout is not None and layer_id in engram_layout.layer_ids:
self.engram = Engram(
@@ -2759,6 +2840,25 @@ class DeepseekV4DecoderLayer(nn.Module):
)
self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data)
def _input_norm(
self,
hidden_states: torch.Tensor,
allow_aiter_quant: bool = True,
coefficients=None,
) -> Tuple[torch.Tensor, Optional[Tuple]]:
"""input_layernorm(hidden_states) as (the bf16 norm attention reads, the pre-quantized
operand of its dense projections or None); coefficients is a ROCm boundary's pending
reduce + sinkhorn."""
if _is_hip:
return _hip.input_norm(
self,
hidden_states,
allow_aiter_quant,
coefficients,
_fused_rmsnorm_fp8_quant,
)
return self.input_layernorm(hidden_states), None
def hc_pre(
self,
x: torch.Tensor,
@@ -2959,6 +3059,11 @@ class DeepseekV4DecoderLayer(nn.Module):
):
return mhc_post_split_h(x, residual, post, comb)
if _is_hip:
y = _hip.hc_post(self, x, residual, post, comb)
if y is not None:
return y
if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
from flashinfer.mhc import mhc_post
@@ -3053,15 +3158,9 @@ class DeepseekV4DecoderLayer(nn.Module):
# Triton fused post+pre (gfx95 small-batch or gfx1250) returns
# norm_fused=False — the input layernorm is NOT folded.
# gfx95 takes the fp8-quant path; gfx1250 takes plain layernorm.
if _use_aiter and _is_gfx95_supported:
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
hidden_states,
self.input_layernorm.weight,
self.rms_norm_eps,
)
else:
hidden_states = self.input_layernorm(hidden_states)
x_quant = None
hidden_states, x_quant = self._input_norm(
hidden_states, allow_aiter_quant=_is_gfx95_supported
)
else:
x_quant = None
else:
@@ -3078,15 +3177,10 @@ class DeepseekV4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
if not norm_fused:
if _use_aiter and _is_gfx95_supported:
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
hidden_states,
self.input_layernorm.weight,
self.rms_norm_eps,
)
else:
hidden_states = self.input_layernorm(hidden_states)
x_quant = None
# gfx1250 takes plain layernorm here
hidden_states, x_quant = self._input_norm(
hidden_states, allow_aiter_quant=_is_gfx95_supported
)
else:
x_quant = None
else:
@@ -3100,15 +3194,10 @@ class DeepseekV4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
if not norm_fused:
if _use_aiter and _is_gfx95_supported:
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
hidden_states,
self.input_layernorm.weight,
self.rms_norm_eps,
)
else:
hidden_states = self.input_layernorm(hidden_states)
x_quant = None
# gfx1250 takes plain layernorm here
hidden_states, x_quant = self._input_norm(
hidden_states, allow_aiter_quant=_is_gfx95_supported
)
else:
x_quant = None
@@ -3290,10 +3379,16 @@ class DeepseekV4DecoderLayer(nn.Module):
if (
x.is_cuda
and torch.version.cuda is not None
and (
get_platform().is_blackwell
or (get_platform().is_sm90 and x.shape[0] == 1)
# gfx950: the fused Triton port wins at every row count (MI350X)
_is_gfx95_supported
or (
torch.version.cuda is not None
and (
get_platform().is_blackwell
or (get_platform().is_sm90 and x.shape[0] == 1)
)
)
)
and x.dtype == torch.bfloat16
):
@@ -3944,15 +4039,7 @@ class DeepseekV4DecoderLayer(nn.Module):
forward_batch=forward_batch,
)
if not norm_fused:
if _use_aiter and (_is_gfx95_supported or _is_gfx1250_supported):
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
hidden_states,
self.input_layernorm.weight,
self.rms_norm_eps,
)
else:
hidden_states = self.input_layernorm(hidden_states)
x_quant = None
hidden_states, x_quant = self._input_norm(hidden_states)
else:
x_quant = None
@@ -4433,14 +4520,20 @@ class DeepseekV4Model(nn.Module):
precomputed_attn = None
combined_attn = None
normalized_attn = None
# HIP: the fused boundary hands the next layer its FFN hc_post unapplied
pending_post = None
for i in range(self.start_layer, self.end_layer):
if tail is not None and i == self.late_layer_start:
combined_attn = None
normalized_attn = None
# Decode reaches back at most SWA_WINDOW positions.
saved_full = attn_backend.enter_late_layer_tail(forward_batch)
hidden_states, prev_pre, input_ids, input_ids_global = (
tail.rows(hidden_states),
if pending_post is not None:
# the fused boundary carries the residual stream here, not in hidden_states
pending_post = tuple(tail.rows(t) for t in pending_post)
else:
hidden_states = tail.rows(hidden_states)
prev_pre, input_ids, input_ids_global = (
tail.rows(prev_pre),
tail.rows(input_ids),
tail.rows(input_ids_global),
@@ -4484,10 +4577,13 @@ class DeepseekV4Model(nn.Module):
next_input = []
# The next layer can consume a collapsed input only if no Engram
# or row selection changes the residual between the two layers.
# The HIP fused boundary carries the residual in pending_post and
# leaves hidden_states None; it never consumes a collapsed input.
next_combined = (
[]
if (
self.config.model_type == "deepseek_v41"
hidden_states is not None
and self.config.model_type == "deepseek_v41"
and (
128 <= hidden_states.shape[0] <= 384
or (
@@ -4505,10 +4601,10 @@ class DeepseekV4Model(nn.Module):
next_norm = self.layers[i + 1].input_layernorm
if (
self.config.model_type == "deepseek_v41"
and get_platform().is_blackwell
and i + 1 < self.end_layer
and tail is None
and hidden_states.is_cuda
and get_platform().is_blackwell
and 0 < hidden_states.shape[0] <= 8
and (
forward_batch.forward_mode.is_decode()
@@ -4533,6 +4629,24 @@ class DeepseekV4Model(nn.Module):
):
next_norm = norm
with ctx:
if _is_hip and self.layers[i].hc_boundary_fused:
hidden_states, prev_pre, pending_post = (
_hip.forward_layer_fused_boundary(
self,
i,
positions=positions,
hidden_states=hidden_states,
input_ids=input_ids,
forward_batch=forward_batch,
input_ids_global=input_ids_global,
prev_pre=prev_pre,
pending_post=pending_post,
capture_dspark=capture_dspark,
)
)
# the fused boundary never publishes a collapsed input for the next layer
precomputed_attn = combined_attn = normalized_attn = None
continue
hidden_states, prev_pre = self.layers[i].forward_hc_pre_from_prev(
positions=positions,
hidden_states=hidden_states,
@@ -335,10 +335,12 @@ class DSparkWorkerV2(BaseSpecWorker):
and not get_parallel().enable_dp_attention
and self.model_runner.pp_size == 1
)
# ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts
# need the group's pynccl communicator
if (
(self._verify_planner.is_compact_mode or static_epilogue_supported)
and self._decode_graph_allowed
and is_cuda()
and (is_cuda() or (is_cuda_alike() and target_is_dsv41))
):
self._verify_epilogue = DsparkVerifyEpilogue(
max_bs=max(get_exec().graph.cuda_graph_config.decode.bs),