dsv4.1-amd: serve DeepSeek-V4.1 on gfx950 (#41308)
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
This commit is contained in:
@@ -26,7 +26,12 @@ def _mhc_post_split_h_kernel(X, R, P, C, Y, H: tl.constexpr, B: tl.constexpr):
|
||||
|
||||
|
||||
def mhc_post_split_h(
|
||||
x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post: torch.Tensor,
|
||||
comb: torch.Tensor,
|
||||
*,
|
||||
block_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Same result as the TileLang post kernel for contiguous BF16 HC=4 inputs."""
|
||||
assert x.dtype == residual.dtype == torch.bfloat16
|
||||
@@ -34,7 +39,7 @@ def mhc_post_split_h(
|
||||
assert residual.shape == (x.shape[0], 4, x.shape[1])
|
||||
assert all(t.is_contiguous() for t in (x, residual, post, comb))
|
||||
output = torch.empty_like(residual)
|
||||
block = 128 if x.shape[0] <= 8 else 1024
|
||||
block = block_size if block_size is not None else (128 if x.shape[0] <= 8 else 1024)
|
||||
_mhc_post_split_h_kernel[(x.shape[0], triton.cdiv(x.shape[1], block))](
|
||||
x,
|
||||
residual,
|
||||
|
||||
@@ -482,8 +482,10 @@ def _block_quant_stack_applies(*, wkv_linears: list[torch.nn.Module]) -> bool:
|
||||
if not (block_quant and hasattr(quant_method, "w8a8_block_fp8_linear")):
|
||||
return False
|
||||
block_out = quant_method.quant_config.weight_block_size[0]
|
||||
# the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout
|
||||
return all(
|
||||
linear.weight.dtype == torch.float8_e4m3fn
|
||||
and linear.weight.dim() == 2
|
||||
and linear.weight.shape[0] % block_out == 0
|
||||
for linear in wkv_linears
|
||||
)
|
||||
@@ -494,7 +496,8 @@ def _dequant_supported(linear: torch.nn.Module) -> bool:
|
||||
weight = linear.weight
|
||||
if weight.dtype in (torch.bfloat16, torch.float16, torch.float32):
|
||||
return True
|
||||
if weight.dtype != torch.float8_e4m3fn:
|
||||
# the gfx950 native MXFP8 route keeps the weight in a 3-D lane-order layout
|
||||
if weight.dtype != torch.float8_e4m3fn or weight.dim() != 2:
|
||||
return False
|
||||
block = 128
|
||||
out_dim, in_dim = weight.shape
|
||||
|
||||
@@ -12,7 +12,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.utils.common import is_npu
|
||||
from sglang.srt.utils.common import is_gfx95_supported, is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -182,7 +182,13 @@ def validate_deepseek_v41_features(server_args: ServerArgs) -> None:
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
incompatible = (
|
||||
("non-CUDA hardware", not get_platform().is_cuda),
|
||||
(
|
||||
"hardware other than CUDA or gfx950",
|
||||
not (
|
||||
get_platform().is_cuda
|
||||
or (get_platform().is_hip and is_gfx95_supported())
|
||||
),
|
||||
),
|
||||
(
|
||||
"prefill CUDA graphs",
|
||||
cfg.cuda_graph_config.prefill.backend != Backend.DISABLED,
|
||||
|
||||
@@ -92,7 +92,7 @@ class ExecFeatures(msgspec.Struct):
|
||||
enable_encoder_swa_bounded_replay: A[
|
||||
bool,
|
||||
"DeepSeek-V4.1 encoder SWA bounded replay: cache Main KV and Indexer keys only, "
|
||||
"rebuild request-owned SWA windows on prefix hits. Experimental; CUDA only.",
|
||||
"rebuild request-owned SWA windows on prefix hits. Experimental; CUDA and gfx950.",
|
||||
] = False
|
||||
enable_decoder_swa_bounded_replay: A[
|
||||
bool,
|
||||
|
||||
@@ -922,7 +922,9 @@ class Envs:
|
||||
# output columns ride along nearly free.
|
||||
SGLANG_ROCM_K3_FUSE_KDA_INPROJ = EnvBool(True)
|
||||
SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256)
|
||||
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang")
|
||||
# ROCm decode attention kernel: auto (aiter_sparse on gfx950, tilelang elsewhere) |
|
||||
# aiter_sparse | tilelang | triton | torch | comparison | unified_kv_triton
|
||||
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("auto")
|
||||
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
|
||||
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
|
||||
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
|
||||
|
||||
@@ -2933,6 +2933,15 @@ class DeepseekV4AttnBackend(
|
||||
# out_loc is -1 for an incomplete group and 0 for padding;
|
||||
# the kernel suppresses both stores.
|
||||
assert out_loc is not None
|
||||
if pool.low_ratio_index_k_is_split(layer_id):
|
||||
# ROCm keeps the index-K payload and scales in two buffers
|
||||
from sglang.srt.layers.attention.dsv4.low_ratio_backend_hip import (
|
||||
store_index_k_split,
|
||||
)
|
||||
|
||||
return store_index_k_split(
|
||||
pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id
|
||||
)
|
||||
index_k_norm_rope_pack_store(
|
||||
indexer.forward_wk(latent),
|
||||
indexer.k_norm.weight.data,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,19 +16,31 @@ from sglang.kernels.ops.attention.dsv4.torch_quant import (
|
||||
fake_quant_fp4,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens
|
||||
from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.utils import add_prefix
|
||||
from sglang.srt.utils import add_prefix, is_gfx95_supported
|
||||
|
||||
|
||||
def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False):
|
||||
if x.is_cuda and torch.version.cuda is not None and x.dtype == torch.bfloat16:
|
||||
def _fused_dsv41_kernels(x: torch.Tensor) -> bool:
|
||||
# validated on CUDA and gfx950; other ROCm GPUs keep the torch forms
|
||||
return x.is_cuda and (torch.version.cuda is not None or is_gfx95_supported())
|
||||
|
||||
|
||||
def _rope_fq4(x, freqs, rope_dim, *, compressed_kv=False, positions=None):
|
||||
"""RoPE plus fake FP4 quantization, fused for bf16 inputs on CUDA and ROCm. With
|
||||
positions, freqs is the whole table and the fused kernel gathers freqs[positions]."""
|
||||
if _fused_dsv41_kernels(x) and x.dtype == torch.bfloat16:
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import (
|
||||
rope_tail_fake_quant_fp4,
|
||||
)
|
||||
|
||||
return rope_tail_fake_quant_fp4(x, freqs, rope_dim, compressed_kv=compressed_kv)
|
||||
return rope_tail_fake_quant_fp4(
|
||||
x, freqs, rope_dim, compressed_kv=compressed_kv, positions=positions
|
||||
)
|
||||
if positions is not None:
|
||||
freqs = freqs[positions]
|
||||
quant = fake_quant_compressed_kv if compressed_kv else fake_quant_fp4
|
||||
return quant(rope_tail(x, freqs, rope_dim))
|
||||
|
||||
@@ -42,9 +54,9 @@ class RMSNorm(nn.Module):
|
||||
self.weight = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# the Triton kernel is bitwise this fallback for bf16 rows on CUDA and ROCm (gfx950)
|
||||
if (
|
||||
x.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
_fused_dsv41_kernels(x)
|
||||
and x.dtype in (torch.bfloat16, torch.float32)
|
||||
and self.weight.dtype in (torch.bfloat16, torch.float32)
|
||||
and x.shape[-1] in (128, 512)
|
||||
@@ -87,11 +99,13 @@ def rope_tail(
|
||||
|
||||
|
||||
def fused_low_ratio_compress_supported() -> bool:
|
||||
"""The fused c1 / c2 / index-K decode kernels pack fp4 with
|
||||
`cvt.rn.satfinite.e2m1x2`, an sm100+ instruction; the answer also fixes the
|
||||
"""The fused c1 / c2 / index-K decode kernels need sm100+ on CUDA (they pack fp4
|
||||
with `cvt.rn.satfinite.e2m1x2`) or gfx950 on ROCm; the answer also fixes the
|
||||
ratio-2 weight layout (`wkv_gate`, or `wkv` plus `wgate`)."""
|
||||
if not torch.cuda.is_available() or torch.version.hip is not None:
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
if torch.version.hip is not None:
|
||||
return is_gfx95_supported()
|
||||
return torch.cuda.get_device_capability()[0] >= 10
|
||||
|
||||
|
||||
@@ -208,6 +222,10 @@ class DeepseekV41Indexer(nn.Module):
|
||||
quant_config=None,
|
||||
prefix=add_prefix("weights_proj", prefix),
|
||||
)
|
||||
# -1 off gfx950 or for shapes the split-K GEMV lacks
|
||||
self.weights_proj_hip_max_tokens = rocm_gemv_split_k_max_tokens(
|
||||
n=self.n_heads, k=config.hidden_size, weight_dtype=torch.bfloat16
|
||||
)
|
||||
# The decode GEMM matches tiny_gemm's reduction order, not cuBLAS's.
|
||||
self.weights_proj_small_max_m = _small_weights_proj_max_m(
|
||||
self.n_heads, config.hidden_size
|
||||
@@ -236,10 +254,17 @@ class DeepseekV41Indexer(nn.Module):
|
||||
k = self.k_norm(self.forward_wk(latent))
|
||||
return _rope_fq4(k, freqs, self.rope_head_dim)
|
||||
|
||||
def queries(self, q_lora: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
||||
def queries(
|
||||
self,
|
||||
q_lora: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""freqs per token, or the whole table with positions (gathered inside the
|
||||
fused RoPE launch)."""
|
||||
q, _ = self.wq_b(q_lora)
|
||||
q = q.view(q.shape[0], self.n_local_heads, self.index_head_dim)
|
||||
return _rope_fq4(q, freqs, self.rope_head_dim)
|
||||
return _rope_fq4(q, freqs, self.rope_head_dim, positions=positions)
|
||||
|
||||
def head_weights_raw(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""`weights_proj(x)` before the scale, [tokens, n_heads] bf16."""
|
||||
|
||||
@@ -0,0 +1,611 @@
|
||||
"""DeepSeek V4.1 low-ratio (1 / 2) indexer on ROCm: FlyDSL fp4 paged MQA logits over the split
|
||||
payload / scale index-K pools, then the AOT top-k transform -- the DeepGEMM path's contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4 import topk_transform_paged
|
||||
from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import (
|
||||
CandidateBlocks,
|
||||
cat_candidate_blocks,
|
||||
select_candidate_blocks_hip,
|
||||
slice_candidate_blocks,
|
||||
topk_transform_paged_sorted,
|
||||
topk_within_candidate_blocks_hip,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
FP4DecodeWorkspace,
|
||||
FP4PrefillWorkspace,
|
||||
aiter_fp4_paged_mqa_logits,
|
||||
index_q_rope_pack_weights_flydsl,
|
||||
indexer_head_weights,
|
||||
logits_rows_per_chunk,
|
||||
pack_fp4_query_flydsl,
|
||||
prepare_fp4_decode_workspace,
|
||||
prepare_fp4_prefill_workspace,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import (
|
||||
index_k_norm_rope_pack_store_split,
|
||||
)
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
_as_int_list,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.attention.dsv4.metadata import PagedIndexerMetadata
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
|
||||
# fp32 logits scored per candidate-block chunk stay under this many bytes
|
||||
_CANDIDATE_SCORE_BUDGET_BYTES = 1 << 30
|
||||
|
||||
|
||||
def _gemv_head_weight_rows(indexer, x: torch.Tensor) -> bool:
|
||||
"""Row counts indexer_head_weights / the split-K GEMV serve: a small contiguous bf16 batch."""
|
||||
return (
|
||||
0 < x.shape[0] <= indexer.weights_proj_hip_max_tokens
|
||||
and x.dim() == 2
|
||||
and x.dtype == torch.bfloat16
|
||||
and x.stride(1) == 1
|
||||
)
|
||||
|
||||
|
||||
def _indexer_head_weights(indexer, x: torch.Tensor) -> torch.Tensor:
|
||||
"""indexer.head_weights(x) as the contiguous bf16 [T, H] the FlyDSL kernels take; decode row
|
||||
counts run indexer_head_weights (same two roundings as aiter's GEMM plus the scale)."""
|
||||
if _gemv_head_weight_rows(indexer, x):
|
||||
return indexer_head_weights(
|
||||
x, indexer.weights_proj.weight, indexer.head_weight_scale
|
||||
)
|
||||
return indexer.head_weights(x).contiguous()
|
||||
|
||||
|
||||
def _indexer_inputs(layer, x, q_lora, pos):
|
||||
"""(payload, scale) query in the FlyDSL layout and the pre-scaled head weights."""
|
||||
indexer = layer.indexer
|
||||
# The kernel sums head scores locally, so the indexer heads must be replicated.
|
||||
assert indexer.n_local_heads == indexer.n_heads
|
||||
if (
|
||||
_gemv_head_weight_rows(indexer, x)
|
||||
and indexer.n_heads % 16 == 0
|
||||
and indexer.n_heads <= 64
|
||||
and indexer.index_head_dim == 128
|
||||
and layer.freqs_cis.dtype == torch.complex64
|
||||
):
|
||||
# decode rows: wq_b, split-K head-weight GEMV, then one launch for RoPE, fp4 pack and reduce
|
||||
q, _ = indexer.wq_b(q_lora)
|
||||
partials = rocm_router_gemv_split_k(x, indexer.weights_proj.weight)
|
||||
return index_q_rope_pack_weights_flydsl(
|
||||
q,
|
||||
layer.freqs_cis,
|
||||
pos,
|
||||
indexer.rope_head_dim,
|
||||
partials,
|
||||
indexer.head_weight_scale,
|
||||
num_heads=indexer.n_heads,
|
||||
)
|
||||
# [T, H, 128] fp4 grid; the RoPE launch gathers freqs_cis[pos] itself
|
||||
q = indexer.queries(q_lora, layer.freqs_cis, positions=pos)
|
||||
q_fp4, q_scale = pack_fp4_query_flydsl(q)
|
||||
weights = _indexer_head_weights(indexer, x) # [T, H] bf16, already scaled
|
||||
return q_fp4, q_scale, weights
|
||||
|
||||
|
||||
def build_low_ratio_decode_workspaces(
|
||||
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
|
||||
) -> Dict[int, FP4DecodeWorkspace]:
|
||||
"""Capture-safe: everything the schedule kernel touches is pinned in the workspace."""
|
||||
return {
|
||||
ratio: prepare_fp4_decode_workspace(
|
||||
meta.page_table,
|
||||
meta.compressed_seq_lens,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
)
|
||||
for ratio, meta in metadata_by_ratio.items()
|
||||
}
|
||||
|
||||
|
||||
def refresh_low_ratio_prefill_workspaces(
|
||||
metadata_by_ratio: Dict[int, PagedIndexerMetadata],
|
||||
previous: Optional[Dict[int, FP4PrefillWorkspace]],
|
||||
) -> Dict[int, FP4PrefillWorkspace]:
|
||||
"""Must run outside CUDA-graph capture; see prepare_fp4_prefill_workspace."""
|
||||
previous = previous or {}
|
||||
return {
|
||||
ratio: prepare_fp4_prefill_workspace(
|
||||
meta.page_table,
|
||||
meta.compressed_seq_lens,
|
||||
workspace=previous.get(ratio),
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
)
|
||||
for ratio, meta in metadata_by_ratio.items()
|
||||
}
|
||||
|
||||
|
||||
def low_ratio_identity_skip_enabled(
|
||||
*, index_topk: int, candidate_topk_blocks: int, candidate_block_size: int
|
||||
) -> bool:
|
||||
"""Whether identity requests may bypass scoring under this model config: the score-free
|
||||
candidate mask is right only while <= index_topk positions fit the candidate top-k."""
|
||||
return triton.cdiv(index_topk, candidate_block_size) <= candidate_topk_blocks
|
||||
|
||||
|
||||
def _fill_identity_request(
|
||||
*,
|
||||
lc: int,
|
||||
slots_j: torch.Tensor,
|
||||
lens: torch.Tensor,
|
||||
page_rows: torch.Tensor,
|
||||
raw_rows: Optional[torch.Tensor],
|
||||
) -> None:
|
||||
"""Rows [t, topk] of the -1 filled buffers: column j < lc holds compressed
|
||||
position j (its slot / its raw index) when j < lens_row, else -1. An identity
|
||||
request publishes no candidates: every reachable block is one."""
|
||||
j = torch.arange(lc, device=lens.device)
|
||||
reach = j[None, :] < lens[:, None]
|
||||
page_rows[:, :lc] = torch.where(reach, slots_j[None, :], -1).to(torch.int32)
|
||||
if raw_rows is not None:
|
||||
raw_rows[:, :lc] = torch.where(reach, j[None, :], -1).to(torch.int32)
|
||||
|
||||
|
||||
def _decode_batch_max_seq_len(forward_batch) -> Optional[int]:
|
||||
"""The host-side longest context of a decode batch, None when unknown."""
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
if seq_lens_cpu is None or seq_lens_cpu.numel() == 0:
|
||||
return None
|
||||
return int(seq_lens_cpu.max().item())
|
||||
|
||||
|
||||
def low_ratio_decode_rows_are_identity(backend, forward_batch, ratio: int) -> bool:
|
||||
"""Decode: every row's compressed context fits index_topk, so nothing needs scoring. Inside a
|
||||
captured graph the answer is the variant being captured; eagerly it is the host-side batch
|
||||
maximum. Target-verify rows keep the scored path."""
|
||||
if not backend.low_ratio_identity_skip or forward_batch is None:
|
||||
return False
|
||||
if not forward_batch.forward_mode.is_decode():
|
||||
return False
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
get_is_capture_mode,
|
||||
skip_low_ratio_indexer,
|
||||
)
|
||||
|
||||
if get_is_capture_mode():
|
||||
return skip_low_ratio_indexer(ratio)
|
||||
max_len = _decode_batch_max_seq_len(forward_batch)
|
||||
return max_len is not None and max_len // ratio <= backend.index_topk
|
||||
|
||||
|
||||
def low_ratio_candidate_span(hf_text_config) -> Optional[int]:
|
||||
"""The span (in positions) within which every block of a request is a
|
||||
candidate, when the model has a candidate source; None otherwise."""
|
||||
# optional HF-config keys: a model without a candidate source lacks them
|
||||
if getattr(hf_text_config, "candidate_source_layer_id", -1) < 0:
|
||||
return None
|
||||
span = getattr(hf_text_config, "candidate_topk_blocks", 0) * getattr(
|
||||
hf_text_config, "candidate_block_size", 0
|
||||
)
|
||||
return span if span > 0 else None
|
||||
|
||||
|
||||
def low_ratio_decode_rows_fit_candidate_span(backend, forward_batch) -> bool:
|
||||
"""Decode: every request's context fits the candidate span, so two-level top-k equals the plain
|
||||
paged top-k. Captured: the variant being captured; eager: the host-side batch maximum."""
|
||||
span = backend.low_ratio_candidate_span
|
||||
if span is None or forward_batch is None:
|
||||
return False
|
||||
if not (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
):
|
||||
return False
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import (
|
||||
get_capture_attention_variant,
|
||||
get_is_capture_mode,
|
||||
)
|
||||
|
||||
if get_is_capture_mode():
|
||||
return get_capture_attention_variant() in (
|
||||
"candidate_all",
|
||||
"candidate_c2_all",
|
||||
"candidate_unfiltered",
|
||||
)
|
||||
max_len = _decode_batch_max_seq_len(forward_batch)
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
if max_len is None:
|
||||
return False
|
||||
max_len += backend.speculative_num_draft_tokens
|
||||
return max_len is not None and max_len <= span
|
||||
|
||||
|
||||
def low_ratio_index_topk_hip_decode(
|
||||
backend, layer, x, q_lora, pos, forward_batch: ForwardBatch
|
||||
) -> None:
|
||||
"""One token per request: paged fp4 logits over every visible compressed slot, level-one
|
||||
candidate blocks where the layer publishes or consumes them, then the top-k transform
|
||||
writes the -1 padded page indices."""
|
||||
pool = backend.token_to_kv_pool
|
||||
metadata = backend.forward_metadata
|
||||
core = metadata.core_metadata
|
||||
ratio = layer.compress_ratio
|
||||
indexer = layer.indexer
|
||||
indexer_metadata = metadata.low_ratio_indexer_metadata(ratio)
|
||||
assert indexer_metadata is not None, f"no decode indexer metadata for {ratio = }"
|
||||
page_indices = core.sparse_page_indices(ratio)
|
||||
raw_indices = core.sparse_raw_indices(ratio)
|
||||
|
||||
if low_ratio_decode_rows_are_identity(backend, forward_batch, ratio):
|
||||
# every row takes the transform's sequential branch (len <= topk), which reads no scores
|
||||
scores = torch.empty(
|
||||
(page_indices.shape[0], 1), dtype=torch.float32, device=pos.device
|
||||
)
|
||||
topk_transform_paged(
|
||||
scores,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
indexer_metadata.page_table,
|
||||
page_indices,
|
||||
indexer_metadata.compressed_page_size,
|
||||
raw_indices,
|
||||
)
|
||||
return
|
||||
|
||||
two_level = indexer.is_candidate_source or indexer.uses_candidates
|
||||
if two_level and low_ratio_decode_rows_fit_candidate_span(backend, forward_batch):
|
||||
# every block is a candidate: the source publishes nothing, every layer runs the plain top-k
|
||||
if indexer.is_candidate_source:
|
||||
backend.candidate_masks = None
|
||||
two_level = False
|
||||
|
||||
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
|
||||
logits = aiter_fp4_paged_mqa_logits(
|
||||
q_fp4=q_fp4,
|
||||
q_scale=q_scale,
|
||||
k_payload=pool.get_index_k_fp4_payload_buffer(layer.layer_id),
|
||||
k_scale=pool.get_index_k_fp4_scale_buffer(layer.layer_id),
|
||||
weights=weights,
|
||||
page_table=indexer_metadata.page_table,
|
||||
c4_seq_lens=indexer_metadata.compressed_seq_lens,
|
||||
weight_scale=1.0,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
is_decode=True,
|
||||
decode_workspace=metadata.fp4_low_ratio_decode_workspaces.get(ratio),
|
||||
)
|
||||
# level one is bounded on device by the compressed lengths: a captured step cannot read them back
|
||||
if two_level and indexer.uses_candidates:
|
||||
candidates = backend.candidate_masks
|
||||
assert (
|
||||
isinstance(candidates, CandidateBlocks)
|
||||
and candidates.ids.shape[0] == logits.shape[0]
|
||||
and candidates.block_size == indexer.candidate_block_size
|
||||
), "candidate blocks missing for decode"
|
||||
topk_within_candidate_blocks_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
candidates,
|
||||
page_table=indexer_metadata.page_table,
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=core.sparse_page_indices(ratio),
|
||||
raw_indices=core.sparse_raw_indices(ratio),
|
||||
sort_output=True,
|
||||
)
|
||||
return
|
||||
if two_level and indexer.is_candidate_source:
|
||||
backend.candidate_masks = select_candidate_blocks_hip(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
topk_blocks=indexer.candidate_topk_blocks,
|
||||
block_size=indexer.candidate_block_size,
|
||||
)
|
||||
topk_transform_paged_sorted(
|
||||
logits,
|
||||
indexer_metadata.compressed_seq_lens,
|
||||
indexer_metadata.page_table,
|
||||
page_indices,
|
||||
indexer_metadata.compressed_page_size,
|
||||
raw_indices,
|
||||
)
|
||||
|
||||
|
||||
def _extend_k_slots(req_to_token, *, ratio, lc_per_req, req_pool_indices, device):
|
||||
"""Per request, the c1/c2 pool slots of its visible compressed positions, and
|
||||
each request's start offset in their concatenation."""
|
||||
starts = [0, *itertools.accumulate(lc_per_req)][:-1]
|
||||
counts = torch.tensor(lc_per_req, dtype=torch.int64, device=device)
|
||||
req_rows = torch.repeat_interleave(req_pool_indices.to(torch.int64), counts)
|
||||
j = torch.arange(int(counts.sum()), device=device) - torch.repeat_interleave(
|
||||
torch.tensor(starts, dtype=torch.int64, device=device), counts
|
||||
)
|
||||
slots = req_to_token[req_rows, j * ratio].to(torch.int64) // ratio
|
||||
return slots, starts
|
||||
|
||||
|
||||
def low_ratio_index_topk_hip_extend(
|
||||
backend,
|
||||
layer,
|
||||
x,
|
||||
q_lora,
|
||||
pos,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> None:
|
||||
"""Ragged prefill: the FlyDSL prefill kernel scores every token's visible compressed positions,
|
||||
candidate masks are published or applied per request, one paged top-k selects every row.
|
||||
Identity requests are written without scores; an all-identity batch skips the kernel."""
|
||||
pool = backend.token_to_kv_pool
|
||||
metadata = backend.forward_metadata
|
||||
core = metadata.core_metadata
|
||||
ratio = layer.compress_ratio
|
||||
indexer = layer.indexer
|
||||
page_indices = core.sparse_page_indices(ratio)
|
||||
raw_indices = core.sparse_raw_indices(ratio)
|
||||
page_indices.fill_(-1)
|
||||
if raw_indices is not None:
|
||||
raw_indices.fill_(-1)
|
||||
|
||||
seq_lens_cpu = _as_int_list(forward_batch.seq_lens_cpu)
|
||||
# bounded replay counts cover only each request's tail
|
||||
if metadata.late_layer_tail is not None:
|
||||
extend_lens_cpu = metadata.late_layer_tail.extend_seq_lens_cpu
|
||||
else:
|
||||
extend_lens_cpu = _as_int_list(forward_batch.extend_seq_lens_cpu)
|
||||
assert seq_lens_cpu is not None and extend_lens_cpu is not None
|
||||
lc_per_req = [s // ratio for s in seq_lens_cpu]
|
||||
if pos.numel() == 0 or not any(lc_per_req):
|
||||
if indexer.is_candidate_source:
|
||||
backend.candidate_masks = [None] * len(lc_per_req)
|
||||
return
|
||||
|
||||
if backend.low_ratio_identity_skip:
|
||||
# identity: every row selects all its visible compressed positions
|
||||
is_identity = [s // ratio <= indexer.index_topk for s in seq_lens_cpu]
|
||||
else:
|
||||
is_identity = [False] * len(seq_lens_cpu)
|
||||
compress_lens = ((pos + 1) // ratio).to(torch.int32)
|
||||
consume = backend.candidate_masks if indexer.uses_candidates else None
|
||||
publish = [] if indexer.is_candidate_source else None
|
||||
if any(is_identity):
|
||||
# only the score-free fill resolves slots itself; scored rows resolve in the top-k transform
|
||||
k_slots, starts = _extend_k_slots(
|
||||
backend.req_to_token,
|
||||
ratio=ratio,
|
||||
lc_per_req=lc_per_req,
|
||||
req_pool_indices=forward_batch.req_pool_indices.to(torch.int64),
|
||||
device=pos.device,
|
||||
)
|
||||
|
||||
def fill_identity_requests(req_lo, req_hi, tok_lo):
|
||||
"""Identity requests req_lo..req_hi: rows written without scores, None published per request."""
|
||||
tok = tok_lo
|
||||
for b in range(req_lo, req_hi):
|
||||
t_len, lc = extend_lens_cpu[b], lc_per_req[b]
|
||||
rows = slice(tok, tok + t_len)
|
||||
tok += t_len
|
||||
if lc == 0:
|
||||
if publish is not None:
|
||||
publish.append(None)
|
||||
continue
|
||||
_fill_identity_request(
|
||||
lc=lc,
|
||||
slots_j=k_slots[starts[b] : starts[b] + lc],
|
||||
lens=compress_lens[rows],
|
||||
page_rows=page_indices[rows],
|
||||
raw_rows=None if raw_indices is None else raw_indices[rows],
|
||||
)
|
||||
if publish is not None:
|
||||
publish.append(None)
|
||||
|
||||
if all(is_identity):
|
||||
fill_identity_requests(0, len(extend_lens_cpu), 0)
|
||||
if publish is not None:
|
||||
backend.candidate_masks = publish
|
||||
return
|
||||
|
||||
indexer_metadata = metadata.low_ratio_indexer_metadata(ratio)
|
||||
assert indexer_metadata is not None, f"no prefill indexer metadata for {ratio = }"
|
||||
num_tokens = pos.shape[0]
|
||||
assert indexer_metadata.page_table.shape[0] >= num_tokens
|
||||
q_fp4, q_scale, weights = _indexer_inputs(layer, x, q_lora, pos)
|
||||
k_payload = pool.get_index_k_fp4_payload_buffer(layer.layer_id)
|
||||
k_scale = pool.get_index_k_fp4_scale_buffer(layer.layer_id)
|
||||
prefill_workspace = metadata.fp4_low_ratio_prefill_workspaces.get(ratio)
|
||||
|
||||
def score_rows(rows: slice) -> torch.Tensor:
|
||||
return aiter_fp4_paged_mqa_logits(
|
||||
q_fp4=q_fp4[rows],
|
||||
q_scale=q_scale[rows],
|
||||
k_payload=k_payload,
|
||||
k_scale=k_scale,
|
||||
weights=weights[rows],
|
||||
page_table=indexer_metadata.page_table[rows],
|
||||
c4_seq_lens=indexer_metadata.compressed_seq_lens[rows],
|
||||
weight_scale=1.0,
|
||||
page_table_bucket=LOW_RATIO_PAGE_TABLE_BUCKET,
|
||||
is_decode=False,
|
||||
prefill_workspace=prefill_workspace if rows.start == 0 else None,
|
||||
)
|
||||
|
||||
def select_rows(
|
||||
rows: slice, req_lo, req_hi, group_is_identity, consume_rows, publish
|
||||
):
|
||||
_select_topk_extend_hip(
|
||||
indexer=indexer,
|
||||
logits=score_rows(rows),
|
||||
lc_per_req=lc_per_req[req_lo:req_hi],
|
||||
extend_lens_cpu=[rows.stop - rows.start]
|
||||
if req_hi == req_lo + 1
|
||||
else extend_lens_cpu[req_lo:req_hi],
|
||||
is_identity=group_is_identity,
|
||||
compress_lens=compress_lens[rows],
|
||||
page_table=indexer_metadata.page_table[rows],
|
||||
page_size=indexer_metadata.compressed_page_size,
|
||||
page_indices=page_indices[rows],
|
||||
raw_indices=raw_indices[rows] if raw_indices is not None else None,
|
||||
consume=consume_rows,
|
||||
publish=publish,
|
||||
)
|
||||
|
||||
# group requests to fit the pooled logits block; rows are independent, so grouping is exact
|
||||
rows_per_chunk = logits_rows_per_chunk(
|
||||
indexer_metadata.page_table, LOW_RATIO_PAGE_TABLE_BUCKET
|
||||
)
|
||||
groups = _request_groups(extend_lens_cpu, rows_per_chunk)
|
||||
for req_lo, req_hi, tok_lo, tok_hi in groups:
|
||||
group_is_identity = is_identity[req_lo:req_hi]
|
||||
if all(group_is_identity):
|
||||
fill_identity_requests(req_lo, req_hi, tok_lo)
|
||||
continue
|
||||
if tok_hi - tok_lo <= rows_per_chunk:
|
||||
select_rows(
|
||||
slice(tok_lo, tok_hi),
|
||||
req_lo,
|
||||
req_hi,
|
||||
group_is_identity,
|
||||
None if consume is None else consume[req_lo:req_hi],
|
||||
publish,
|
||||
)
|
||||
continue
|
||||
# a request wider than the logits block: scored in row chunks, its mask published in one piece
|
||||
assert req_hi == req_lo + 1, (req_lo, req_hi)
|
||||
pieces = []
|
||||
for lo in range(tok_lo, tok_hi, rows_per_chunk):
|
||||
rows = slice(lo, min(lo + rows_per_chunk, tok_hi))
|
||||
piece = [] if publish is not None else None
|
||||
consume_rows = None
|
||||
if consume is not None:
|
||||
consume_rows = [
|
||||
None
|
||||
if consume[req_lo] is None
|
||||
else slice_candidate_blocks(
|
||||
consume[req_lo], slice(lo - tok_lo, rows.stop - tok_lo)
|
||||
)
|
||||
]
|
||||
select_rows(rows, req_lo, req_hi, group_is_identity, consume_rows, piece)
|
||||
if piece:
|
||||
pieces.append(piece[0])
|
||||
if publish is not None:
|
||||
publish.append(cat_candidate_blocks(pieces))
|
||||
if publish is not None:
|
||||
backend.candidate_masks = publish
|
||||
|
||||
|
||||
def _select_topk_extend_hip(
|
||||
*,
|
||||
indexer,
|
||||
logits: torch.Tensor,
|
||||
lc_per_req: List[int],
|
||||
extend_lens_cpu: List[int],
|
||||
is_identity: List[bool],
|
||||
compress_lens: torch.Tensor,
|
||||
page_table: torch.Tensor,
|
||||
page_size: int,
|
||||
page_indices: torch.Tensor,
|
||||
raw_indices: Optional[torch.Tensor],
|
||||
consume: Optional[List[Optional[CandidateBlocks]]],
|
||||
publish: Optional[List[Optional[CandidateBlocks]]],
|
||||
) -> None:
|
||||
"""Row t of logits scores its request's compressed positions in columns 0..lc-1, reachable
|
||||
up to compress_lens[t]. A source layer publishes one CandidateBlocks per request (None
|
||||
for identity / empty), a consumer selects inside its published blocks, others run one paged top-k."""
|
||||
assert page_indices.shape[1] == indexer.index_topk, (
|
||||
f"the paged top-k selects page_indices.shape[1] = {page_indices.shape[1]} "
|
||||
f"slots, the indexer wants {indexer.index_topk}"
|
||||
)
|
||||
if publish is not None:
|
||||
tok_start = 0
|
||||
for b, (lc, t_len) in enumerate(zip(lc_per_req, extend_lens_cpu)):
|
||||
rows = slice(tok_start, tok_start + t_len)
|
||||
tok_start += t_len
|
||||
if lc == 0 or t_len == 0 or is_identity[b]:
|
||||
# Consumers index the publication by request, so keep the slot.
|
||||
publish.append(None)
|
||||
continue
|
||||
step = max(1, _CANDIDATE_SCORE_BUDGET_BYTES // (lc * 4))
|
||||
pieces = []
|
||||
for lo in range(0, t_len, step):
|
||||
chunk = slice(rows.start + lo, rows.start + min(lo + step, t_len))
|
||||
pieces.append(
|
||||
select_candidate_blocks_hip(
|
||||
logits[chunk, :lc],
|
||||
compress_lens[chunk],
|
||||
topk_blocks=indexer.candidate_topk_blocks,
|
||||
block_size=indexer.candidate_block_size,
|
||||
)
|
||||
)
|
||||
publish.append(cat_candidate_blocks(pieces))
|
||||
if consume is not None:
|
||||
tok_start = 0
|
||||
for b, t_len in enumerate(extend_lens_cpu):
|
||||
rows = slice(tok_start, tok_start + t_len)
|
||||
tok_start += t_len
|
||||
if t_len == 0:
|
||||
continue
|
||||
rows_page = page_indices[rows]
|
||||
rows_raw = raw_indices[rows] if raw_indices is not None else None
|
||||
if consume[b] is None:
|
||||
topk_transform_paged_sorted(
|
||||
logits[rows],
|
||||
compress_lens[rows].contiguous(),
|
||||
page_table[rows],
|
||||
rows_page,
|
||||
page_size,
|
||||
rows_raw,
|
||||
)
|
||||
else:
|
||||
topk_within_candidate_blocks_hip(
|
||||
logits[rows],
|
||||
compress_lens[rows],
|
||||
consume[b],
|
||||
page_table=page_table[rows],
|
||||
page_size=page_size,
|
||||
page_indices=rows_page,
|
||||
raw_indices=rows_raw,
|
||||
sort_output=True,
|
||||
)
|
||||
return
|
||||
topk_transform_paged_sorted(
|
||||
logits,
|
||||
compress_lens.contiguous(),
|
||||
page_table,
|
||||
page_indices,
|
||||
page_size,
|
||||
raw_indices,
|
||||
)
|
||||
|
||||
|
||||
def _request_groups(
|
||||
extend_lens_cpu: List[int], rows_per_chunk: int
|
||||
) -> List[Tuple[int, int, int, int]]:
|
||||
"""Consecutive request groups whose token rows fit rows_per_chunk (a single
|
||||
request always forms a group): (req_lo, req_hi, tok_lo, tok_hi)."""
|
||||
groups = []
|
||||
req_lo, tok_lo, tok = 0, 0, 0
|
||||
for r, t_len in enumerate(extend_lens_cpu):
|
||||
if r > req_lo and tok + t_len - tok_lo > rows_per_chunk:
|
||||
groups.append((req_lo, r, tok_lo, tok))
|
||||
req_lo, tok_lo = r, tok
|
||||
tok += t_len
|
||||
groups.append((req_lo, len(extend_lens_cpu), tok_lo, tok))
|
||||
return groups
|
||||
|
||||
|
||||
def store_index_k_split(
|
||||
pool, layer, indexer, latent, freqs_cis, pos, out_loc, layer_id: int
|
||||
) -> None:
|
||||
"""The c1 / c2 decode index-K store into ROCm's split pools: payload and scales in
|
||||
two buffers, where CUDA packs one fused [.., 68]-byte row."""
|
||||
index_k_norm_rope_pack_store_split(
|
||||
indexer.forward_wk(latent),
|
||||
indexer.k_norm.weight.data,
|
||||
indexer.k_norm.eps,
|
||||
freqs_cis,
|
||||
pos,
|
||||
out_loc,
|
||||
pool.get_index_k_fp4_payload_buffer(layer_id),
|
||||
pool.get_index_k_fp4_scale_buffer(layer_id),
|
||||
ratio=layer.compress_ratio,
|
||||
)
|
||||
@@ -98,7 +98,7 @@ def create_dsv41_candidate_graph_variants(
|
||||
import torch
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
|
||||
text_config = model_runner.model_config.hf_text_config
|
||||
dspark_target_verify = (
|
||||
@@ -110,8 +110,11 @@ def create_dsv41_candidate_graph_variants(
|
||||
if not (
|
||||
(capture_forward_mode == ForwardMode.DECODE or dspark_target_verify)
|
||||
and model_runner.device == "cuda"
|
||||
and not is_hip()
|
||||
and torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10
|
||||
and (
|
||||
is_gfx95_supported()
|
||||
if is_hip()
|
||||
else torch.cuda.get_device_capability(model_runner.gpu_id)[0] >= 10
|
||||
)
|
||||
and getattr(text_config, "model_type", None) == "deepseek_v41"
|
||||
and getattr(text_config, "candidate_source_layer_id", -1) >= 0
|
||||
):
|
||||
|
||||
@@ -1,17 +1,207 @@
|
||||
from typing import Any, Optional
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
|
||||
FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn
|
||||
|
||||
|
||||
_HIP_BACKENDS = ("tilelang", "triton", "aiter_sparse", "torch", "comparison")
|
||||
|
||||
# at this many query tokens and above the aiter sparse kernel runs unsplit (batch-invariant prefill)
|
||||
_AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS = 1024
|
||||
|
||||
|
||||
def hip_attn_kv_splits() -> int:
|
||||
"""Split-KV count for the HIP sparse decode kernels. Deterministic inference pins 4
|
||||
splits (a fixed combine order keeps the bits identical at every batch size) and
|
||||
everything else uses 0: adaptive splits plus the native 16-head attention for small
|
||||
TP4 decode batches."""
|
||||
try:
|
||||
deterministic = get_exec().deterministic.enable_deterministic_inference
|
||||
except ValueError: # no published exec config (kernel tests, offline tools)
|
||||
deterministic = False
|
||||
return 4 if deterministic else 0
|
||||
|
||||
|
||||
# the eager warmups before each capture fill this, so captured graphs reuse these tensors
|
||||
_UNIFORM_INDPTR_CACHE: dict[tuple[int, int, str], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _uniform_indptr(num_tokens: int, width: int, device: str) -> torch.Tensor:
|
||||
"""Row pointers of the aiter sparse decode kernel: token t reads kv_indices[t*w : (t+1)*w].
|
||||
|
||||
Only tensors built outside capture are cached: an arange recorded into a graph runs only
|
||||
when that graph replays, and V4.1 captures two verify graphs per batch size."""
|
||||
key = (num_tokens, width, device)
|
||||
indptr = _UNIFORM_INDPTR_CACHE.get(key)
|
||||
if indptr is not None:
|
||||
return indptr
|
||||
indptr = torch.arange(
|
||||
0, (num_tokens + 1) * width, width, dtype=torch.int32, device=device
|
||||
)
|
||||
if (
|
||||
num_tokens < _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
_UNIFORM_INDPTR_CACHE[key] = indptr
|
||||
return indptr
|
||||
|
||||
|
||||
def aiter_sparse_decode_fwd(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
extra_k_cache: Optional[torch.Tensor] = None,
|
||||
extra_indices_in_kvcache: Optional[torch.Tensor] = None,
|
||||
inv_rope: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
topk_length: Optional[torch.Tensor] = None,
|
||||
extra_topk_length: Optional[torch.Tensor] = None,
|
||||
**_,
|
||||
):
|
||||
"""aiter's gfx950 gluon sparse decode kernel (pa_decode_sparse) behind the
|
||||
flash_mla_with_kvcache shapes. Only -1 entries are skipped, so callers fold
|
||||
topk_length into the index lists; inv_rope folds the model's inverse RoPE into the output."""
|
||||
if k_cache.shape[-1] in (
|
||||
KVLayout.V41.bytes_per_token,
|
||||
KVLayout.V41_FP4.bytes_per_token,
|
||||
):
|
||||
from sglang.kernels.ops.attention.dsv4.compact_attention_hip import (
|
||||
compact_attention_hip,
|
||||
)
|
||||
|
||||
b, s, h, d = q.shape
|
||||
out = compact_attention_hip(
|
||||
q.reshape(b * s, h, d),
|
||||
k_cache,
|
||||
indices,
|
||||
topk_length.reshape(-1),
|
||||
attn_sink,
|
||||
extra_cache=extra_k_cache,
|
||||
extra_indices=extra_indices_in_kvcache,
|
||||
extra_lengths=(
|
||||
extra_topk_length.reshape(-1) if extra_k_cache is not None else None
|
||||
),
|
||||
softmax_scale=softmax_scale,
|
||||
inv_rope=inv_rope,
|
||||
splits=(
|
||||
1
|
||||
if b * s >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS
|
||||
else hip_attn_kv_splits() or None
|
||||
),
|
||||
)
|
||||
return out.view(b, s, h, d), None
|
||||
|
||||
from aiter.ops.triton.attention.pa_decode_sparse import pa_decode_sparse
|
||||
|
||||
from sglang.kernels.ops.attention.sparse_decode_reduce_hip import (
|
||||
gfx_sparse_split_reduce,
|
||||
)
|
||||
|
||||
b, s, h, d = q.shape
|
||||
n = b * s
|
||||
q3 = q.reshape(n, h, d)
|
||||
assert k_cache.dim() == 4 and k_cache.shape[2] == 1, k_cache.shape
|
||||
cache = k_cache.view(torch.uint8).squeeze(2)
|
||||
idx = indices.reshape(-1)
|
||||
assert idx.dtype == torch.int32 and idx.is_contiguous()
|
||||
indptr = _uniform_indptr(n, indices.shape[-1], str(q.device))
|
||||
extra_kwargs = {}
|
||||
if extra_k_cache is not None:
|
||||
assert extra_k_cache.dim() == 4 and extra_k_cache.shape[2] == 1
|
||||
extra_idx = extra_indices_in_kvcache.reshape(-1)
|
||||
assert extra_idx.dtype == torch.int32 and extra_idx.is_contiguous()
|
||||
extra_kwargs = dict(
|
||||
extra_cache=extra_k_cache.view(torch.uint8).squeeze(2),
|
||||
extra_indices=extra_idx,
|
||||
extra_indptr=_uniform_indptr(
|
||||
n, extra_indices_in_kvcache.shape[-1], str(q.device)
|
||||
),
|
||||
)
|
||||
if n >= _AITER_SPARSE_SINGLE_SPLIT_MIN_TOKENS:
|
||||
extra_kwargs["kv_splits"] = 1
|
||||
elif hip_attn_kv_splits() > 0:
|
||||
# a pinned split count keeps the combine order, hence the bits, the same at every batch size
|
||||
extra_kwargs["kv_splits"] = hip_attn_kv_splits()
|
||||
out = pa_decode_sparse(
|
||||
q3,
|
||||
cache,
|
||||
idx,
|
||||
indptr,
|
||||
attn_sink,
|
||||
softmax_scale,
|
||||
skip_reduce=True,
|
||||
**extra_kwargs,
|
||||
)
|
||||
if isinstance(out, tuple):
|
||||
# Split-KV partials (acc, m, l): combine them here.
|
||||
part_acc, part_m, part_l = out
|
||||
out = gfx_sparse_split_reduce(
|
||||
part_acc, part_m, part_l, attn_sink, q.dtype, inv_rope=inv_rope
|
||||
)
|
||||
elif inv_rope is not None:
|
||||
_apply_inverse_rope(out, inv_rope)
|
||||
return out.view(b, s, h, d), None
|
||||
|
||||
|
||||
def _apply_inverse_rope(
|
||||
out: torch.Tensor, inv_rope: Tuple[torch.Tensor, torch.Tensor]
|
||||
) -> None:
|
||||
"""The model's standalone inverse RoPE on out [n, h, d] (last 64 dims of
|
||||
every head), for the paths that did not fold it into their combine."""
|
||||
from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace
|
||||
|
||||
freqs_real, positions = inv_rope
|
||||
n = out.shape[0]
|
||||
# freqs_real is view_as_real(freqs_cis).flatten(-2); fused_rope_inplace takes the complex table
|
||||
freqs_cis = torch.view_as_complex(freqs_real.view(freqs_real.shape[0], -1, 2))
|
||||
fused_rope_inplace(
|
||||
out.view(n, -1, out.shape[-1])[..., -freqs_real.shape[-1] :],
|
||||
None,
|
||||
freqs_cis,
|
||||
positions.view(-1)[:n],
|
||||
inverse=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_hip_flashmla_backend(backend: Optional[str] = None) -> str:
|
||||
"""The HIP decode attention kernel name; "auto" (the default) is aiter's
|
||||
gluon sparse kernel on gfx950 and the tilelang partial + combine elsewhere."""
|
||||
if backend is None:
|
||||
backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get()
|
||||
if backend == "auto":
|
||||
return "aiter_sparse" if is_gfx95_supported() else "tilelang"
|
||||
if backend == "aiter_sparse" and not is_gfx95_supported():
|
||||
raise ValueError(
|
||||
"SGLANG_HACK_FLASHMLA_BACKEND=aiter_sparse needs gfx950 (aiter's gluon sparse "
|
||||
"decode kernel); use auto or tilelang"
|
||||
)
|
||||
return backend
|
||||
|
||||
|
||||
# kernels taking the real per-rank head count; the tilelang kernel builds only for 64-padded widths
|
||||
_HIP_BACKENDS_ANY_HEAD_COUNT = frozenset({"aiter_sparse", "triton"})
|
||||
|
||||
|
||||
def hip_attention_needs_head_pad() -> bool:
|
||||
"""Whether the kernel DeepseekV4HipRadixBackend.forward picks needs the per-rank query
|
||||
heads padded to 64 (zero q, zero sink)."""
|
||||
return resolve_hip_flashmla_backend() not in _HIP_BACKENDS_ANY_HEAD_COUNT
|
||||
|
||||
|
||||
def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
|
||||
if is_hip():
|
||||
backend = envs.SGLANG_HACK_FLASHMLA_BACKEND.get()
|
||||
# a CUDA backend name (the "comparison" recursion passes "kernel") takes the HIP default
|
||||
backend = resolve_hip_flashmla_backend(
|
||||
backend if backend in _HIP_BACKENDS else None
|
||||
)
|
||||
else:
|
||||
import sgl_kernel.flash_mla as flash_mla
|
||||
|
||||
@@ -44,6 +234,9 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs):
|
||||
|
||||
return triton_fp8_attention_fwd(**kwargs)
|
||||
|
||||
if backend == "aiter_sparse":
|
||||
return aiter_sparse_decode_fwd(**kwargs)
|
||||
|
||||
if backend == "kernel":
|
||||
return flash_mla.flash_mla_with_kvcache(**kwargs)
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.kernels.ops.embeddings.engram_hash import (
|
||||
engram_hash_ids_and_commit,
|
||||
)
|
||||
from sglang.srt.distributed import tensor_model_parallel_all_reduce
|
||||
from sglang.srt.distributed.parallel_state import inplace_all_reduce
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_all_gather_into_tensor,
|
||||
@@ -47,7 +48,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.runtime_context import get_model, get_parallel, get_serving
|
||||
from sglang.srt.utils import add_prefix, is_cuda
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -55,10 +56,13 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_MILLER_RABIN_WITNESSES = (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37)
|
||||
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
def _cuda_kernels(t: torch.Tensor) -> bool:
|
||||
"""True where the Triton kernels apply; ROCm and CPU take the torch paths."""
|
||||
return t.is_cuda and is_cuda()
|
||||
"""True where the Triton kernels apply (CUDA and gfx950); other ROCm GPUs and CPU
|
||||
take the torch paths."""
|
||||
return t.is_cuda and (is_cuda() or is_gfx95_supported())
|
||||
|
||||
|
||||
def _is_prime(n: int) -> bool:
|
||||
@@ -775,9 +779,18 @@ class EngramEmbedding(nn.Module):
|
||||
return self._empty(indices)
|
||||
values = self._owned_rows(indices)
|
||||
if self.tp_size > 1:
|
||||
values = tensor_model_parallel_all_reduce(values)
|
||||
values = self._reduce_owned_rows(values)
|
||||
return values
|
||||
|
||||
def _reduce_owned_rows(self, values: torch.Tensor) -> torch.Tensor:
|
||||
if _is_hip and values.is_cuda:
|
||||
# Integer addition preserves all BF16 bits because exactly one shard owns each row.
|
||||
inplace_all_reduce(
|
||||
values.view(torch.int32), group_name=get_parallel().tp_group.unique_name
|
||||
)
|
||||
return values
|
||||
return tensor_model_parallel_all_reduce(values)
|
||||
|
||||
def _empty(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
*indices.shape, self.dim, dtype=torch.bfloat16, device=indices.device
|
||||
@@ -833,9 +846,14 @@ class EngramEmbedding(nn.Module):
|
||||
and self.tp_size == get_parallel().attn_dp_size
|
||||
and rows == self.tp_size * local.shape[0]
|
||||
) or is_dp_gatherv_active():
|
||||
dp_reduce_scatter_tensor(local, values)
|
||||
if _is_hip and values.is_cuda:
|
||||
dp_reduce_scatter_tensor(
|
||||
local.view(torch.int32), values.view(torch.int32)
|
||||
)
|
||||
else:
|
||||
dp_reduce_scatter_tensor(local, values)
|
||||
else:
|
||||
dp_scatter(local, tensor_model_parallel_all_reduce(values), forward_batch)
|
||||
dp_scatter(local, self._reduce_owned_rows(values), forward_batch)
|
||||
return local.view(*indices.shape, self.dim)
|
||||
|
||||
|
||||
|
||||
@@ -233,6 +233,8 @@ if _use_aiter:
|
||||
from aiter.fused_moe import fused_topk as aiter_fused_topk
|
||||
except ImportError:
|
||||
raise ImportError("aiter is required when SGLANG_USE_AITER is set to True")
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_reduce_partials
|
||||
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_gate
|
||||
if _is_musa:
|
||||
try:
|
||||
from mate import moe_fused_gate
|
||||
@@ -641,7 +643,10 @@ class TopK(BaseFusedOp):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
if router_logits_partials is not None:
|
||||
rocm_router_reduce_partials(router_logits_partials, router_logits)
|
||||
self.topk_config.torch_native = True
|
||||
topk_output = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
@@ -662,7 +667,11 @@ class TopK(BaseFusedOp):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
|
||||
fixed-order sum is the logits; router_logits is then a buffer the fused gate
|
||||
fills, and any other reader first reduces the partials into it."""
|
||||
if dynamic_expert_bias is not None:
|
||||
output_format = TopKOutputFormat.STANDARD
|
||||
elif self.topk_config.output_format is not None:
|
||||
@@ -688,6 +697,13 @@ class TopK(BaseFusedOp):
|
||||
else:
|
||||
output_format = TopKOutputFormat.STANDARD
|
||||
|
||||
if (
|
||||
router_logits_partials is not None
|
||||
and output_format != TopKOutputFormat.STANDARD
|
||||
):
|
||||
rocm_router_reduce_partials(router_logits_partials, router_logits)
|
||||
router_logits_partials = None
|
||||
|
||||
if output_format == TopKOutputFormat.TRITON_KERNEL:
|
||||
# renormalize=True is equivalent to sm_first=False
|
||||
(
|
||||
@@ -729,6 +745,7 @@ class TopK(BaseFusedOp):
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
dynamic_expert_bias=dynamic_expert_bias,
|
||||
router_logits_partials=router_logits_partials,
|
||||
)
|
||||
return self._apply_waterfill(topk_output, hidden_states.shape[0])
|
||||
|
||||
@@ -1423,11 +1440,23 @@ def biased_topk_jit_kernel_impl(
|
||||
apply_routed_scaling_factor_on_output: Optional[bool] = False,
|
||||
packed_out: Optional[torch.Tensor] = None,
|
||||
sqrtsoftplus_log1p: bool = False,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
|
||||
|
||||
if _use_aiter and scoring_func == "sqrtsoftplus" and num_fused_shared_experts == 0:
|
||||
assert packed_out is None, "aiter topk_gating cannot emit packed ids"
|
||||
if router_logits_partials is not None:
|
||||
# ROCm decode router: split-K reduce + gate in one launch
|
||||
return rocm_router_gate(
|
||||
gating_output,
|
||||
correction_bias,
|
||||
topk,
|
||||
renormalize,
|
||||
routed_scaling_factor,
|
||||
partials=router_logits_partials,
|
||||
)
|
||||
|
||||
from aiter import topk_gating
|
||||
|
||||
num_tokens = gating_output.shape[0]
|
||||
@@ -1451,6 +1480,7 @@ def biased_topk_jit_kernel_impl(
|
||||
return topk_weights, topk_ids
|
||||
|
||||
else:
|
||||
assert router_logits_partials is None
|
||||
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
|
||||
|
||||
# DeepSeek-V4 stores e_score_correction_bias in bf16 (for the aiter
|
||||
@@ -2460,6 +2490,7 @@ def select_experts(
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> StandardTopKOutput:
|
||||
top_k = topk_config.top_k
|
||||
use_grouped_topk = topk_config.use_grouped_topk
|
||||
@@ -2495,6 +2526,22 @@ def select_experts(
|
||||
)
|
||||
routing_overridden = simulate_uniform_experts or simulate_round_robin_experts
|
||||
|
||||
if router_logits_partials is not None and not (
|
||||
_use_aiter
|
||||
and scoring_func == "sqrtsoftplus"
|
||||
and custom_routing_function is None
|
||||
and not use_grouped_topk
|
||||
and not torch_native
|
||||
and expert_location_dispatch_info is None
|
||||
and (
|
||||
num_fused_shared_experts == 0
|
||||
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
)
|
||||
):
|
||||
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
|
||||
rocm_router_reduce_partials(router_logits_partials, router_logits)
|
||||
router_logits_partials = None
|
||||
|
||||
(
|
||||
router_logits,
|
||||
correction_bias,
|
||||
@@ -2619,6 +2666,8 @@ def select_experts(
|
||||
_packed_kwargs = dict(packed_out=packed_topk)
|
||||
if topk_config.sqrtsoftplus_log1p:
|
||||
_packed_kwargs["sqrtsoftplus_log1p"] = True
|
||||
if router_logits_partials is not None:
|
||||
_packed_kwargs["router_logits_partials"] = router_logits_partials
|
||||
topk_weights, topk_ids = _biased_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
|
||||
@@ -126,6 +126,9 @@ _is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_cpu = is_cpu()
|
||||
_is_fp8_fnuz = is_fp8_fnuz()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
if _is_hip:
|
||||
from sglang.srt.layers.quantization import fp8_hip
|
||||
# gfx942 (MI300) has no MX matmul HW; MXFP8 checkpoints are converted to
|
||||
# block-fp8 [128,128] at load and run through the native block-fp8 kernels.
|
||||
# SGLANG_FORCE_MXFP8_BLOCK_CONVERT=1 opts into that same block-fp8 path on
|
||||
@@ -951,6 +954,8 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
"weight_scale_inv_swizzled",
|
||||
block_scale_interleave(scale_u8.contiguous()).contiguous(),
|
||||
)
|
||||
elif backend.is_gfx95():
|
||||
fp8_hip.process_dense_weights(self, layer, scale_u8)
|
||||
elif backend.is_deep_gemm():
|
||||
from sglang.srt.layers.deep_gemm_wrapper.configurer import (
|
||||
DEEPGEMM_SCALE_UE8M0,
|
||||
@@ -1162,6 +1167,9 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
if _is_hip and self.block_fp8_as_mxfp8 and self.mxfp8_dense_backend.is_gfx95():
|
||||
return fp8_hip.apply_dense(self, layer, x, bias)
|
||||
|
||||
mxfp8_view = self.use_mxfp8 or (
|
||||
self.block_fp8_as_mxfp8 and layer.block_fp8_mxfp8_ready
|
||||
)
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""gfx950 dense routes of Fp8LinearMethod for 32x32-block fp8 checkpoints served as
|
||||
MXFP8 (block_fp8_as_mxfp8): the Triton dot_scaled kernel, or the native scaled-MFMA
|
||||
kernels on a lane-ordered weight. The fused gfx950 producers hand these routes their
|
||||
operand as an Fp8GridActivation (bf16 already on the fp8 grid) or an Mxfp8Activation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
|
||||
Fp8GridActivation,
|
||||
Mxfp8Activation,
|
||||
bf16_dequant_blockscaled_linear,
|
||||
dequant_block_fp8_weight_to_bf16,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import (
|
||||
native_route_supports,
|
||||
prepare_mxfp8_native_weight,
|
||||
)
|
||||
from sglang.srt.layers.utils import copy_or_rebind_param
|
||||
|
||||
|
||||
def process_dense_weights(method, layer: torch.nn.Module, scale_u8) -> None:
|
||||
"""The gfx950 branch of Fp8LinearMethod.process_weights_after_loading."""
|
||||
backend = method.mxfp8_dense_backend
|
||||
if backend.is_gfx95_dot_scaled():
|
||||
# dot_scaled reads canonical [N, K // 32] e8m0 bytes; block scales stay for direct readers
|
||||
if scale_u8 is not None:
|
||||
copy_or_rebind_param(layer, "weight_scale_inv_mx", scale_u8.contiguous())
|
||||
return
|
||||
assert backend.is_gfx95_mxfp8_native()
|
||||
n, k = layer.weight.shape
|
||||
layer.mxfp8_native_ready = False
|
||||
if native_route_supports(n, k):
|
||||
# same bytes in scaled-MFMA lane order
|
||||
shuffled, scale_ue8m0 = prepare_mxfp8_native_weight(
|
||||
layer.weight.data,
|
||||
layer.weight_scale_inv.data,
|
||||
method.weight_block_size,
|
||||
)
|
||||
copy_or_rebind_param(layer, "weight", shuffled.view(torch.float8_e4m3fn))
|
||||
copy_or_rebind_param(layer, "weight_scale_mx_e8m0", scale_ue8m0)
|
||||
layer.mxfp8_native_ready = True
|
||||
else:
|
||||
# a shape the native kernels do not tile keeps the bf16-dequant route
|
||||
copy_or_rebind_param(
|
||||
layer,
|
||||
"weight_bf16",
|
||||
dequant_block_fp8_weight_to_bf16(
|
||||
layer.weight.data,
|
||||
layer.weight_scale_inv.data,
|
||||
method.weight_block_size,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def apply_dense(
|
||||
method, layer: torch.nn.Module, x, bias: Optional[torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
"""Fp8LinearMethod.apply on a gfx950 route. x is a bf16 tensor, an (fp8, scale)
|
||||
tuple from a fused quant kernel, or one of the wrappers the fused producers emit."""
|
||||
backend = method.mxfp8_dense_backend
|
||||
mxfp8_ready = layer.block_fp8_mxfp8_ready
|
||||
native_route = mxfp8_ready and backend.is_gfx95_mxfp8_native()
|
||||
# Unwrap the producer's operand: the native route takes fp8 + scales or the fp8-grid
|
||||
# bf16 directly; the dot_scaled route quantizes the plain tensor itself (per-32
|
||||
# rounding is idempotent, so the wrapper's rounding is exact for it).
|
||||
input_scale, on_fp8_grid = None, False
|
||||
if isinstance(x, Mxfp8Activation):
|
||||
if native_route:
|
||||
x, input_scale = x.q, x.scale
|
||||
else:
|
||||
x = dequant_mxfp8_to_bf16(x.q, x.scale)
|
||||
elif isinstance(x, Fp8GridActivation):
|
||||
x, on_fp8_grid = x.x, native_route
|
||||
elif isinstance(x, tuple):
|
||||
x, input_scale = x
|
||||
if native_route:
|
||||
return _apply_native(method, layer, x, bias, input_scale, on_fp8_grid)
|
||||
if mxfp8_ready and input_scale is None:
|
||||
return method.w8a8_mxfp8_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale_inv_mx,
|
||||
input_scale=None,
|
||||
bias=bias,
|
||||
)
|
||||
# a shape the gfx950 kernels do not tile, or a pre-quantized tuple off the native route
|
||||
return method.w8a8_block_fp8_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
block_size=method.weight_block_size,
|
||||
weight_scale=layer.weight_scale_inv,
|
||||
input_scale=input_scale,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
|
||||
def _apply_native(
|
||||
method,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor],
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
input_on_fp8_grid: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""The native route (mxfp8_native_amd_gfx95); a layer whose shape the native
|
||||
kernels do not tile keeps the bf16-dequant route."""
|
||||
if layer.mxfp8_native_ready:
|
||||
# an fp8-grid bf16 input re-encodes exactly, so it needs no flag here
|
||||
return method.w8a8_mxfp8_linear(
|
||||
input=x,
|
||||
weight_shuffled=layer.weight.view(torch.uint8),
|
||||
weight_scale_ue8m0=layer.weight_scale_mx_e8m0,
|
||||
input_scale=input_scale,
|
||||
bias=bias,
|
||||
)
|
||||
if input_scale is not None:
|
||||
x = dequant_mxfp8_to_bf16(x, input_scale)
|
||||
input_on_fp8_grid = True
|
||||
return bf16_dequant_blockscaled_linear(
|
||||
x, layer.weight_bf16, bias=bias, input_on_fp8_grid=input_on_fp8_grid
|
||||
)
|
||||
@@ -381,6 +381,8 @@ class Mxfp8DenseGemmBackend(Enum):
|
||||
FLASHINFER_TRTLLM = "flashinfer_trtllm"
|
||||
DEEP_GEMM = "deep_gemm"
|
||||
GFX95_DOT_SCALED = "gfx95_dot_scaled"
|
||||
# gfx950 native MXFP8: lane-ordered fp8 weight + ue8m0 scale bytes; untiled shapes keep bf16
|
||||
GFX95_MXFP8_NATIVE = "gfx95_mxfp8_native"
|
||||
UNSUPPORTED = "unsupported"
|
||||
|
||||
def is_flashinfer_cutlass(self) -> bool:
|
||||
@@ -401,6 +403,15 @@ class Mxfp8DenseGemmBackend(Enum):
|
||||
def is_gfx95_dot_scaled(self) -> bool:
|
||||
return self == Mxfp8DenseGemmBackend.GFX95_DOT_SCALED
|
||||
|
||||
def is_gfx95_mxfp8_native(self) -> bool:
|
||||
return self == Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE
|
||||
|
||||
def is_gfx95(self) -> bool:
|
||||
return self in (
|
||||
Mxfp8DenseGemmBackend.GFX95_DOT_SCALED,
|
||||
Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE,
|
||||
)
|
||||
|
||||
def is_unsupported(self) -> bool:
|
||||
return self == Mxfp8DenseGemmBackend.UNSUPPORTED
|
||||
|
||||
@@ -724,8 +735,14 @@ def _unsupported_mxfp8_linear(*args, **kwargs) -> torch.Tensor:
|
||||
|
||||
|
||||
def resolve_block_fp8_mxfp8_backend() -> Mxfp8DenseGemmBackend:
|
||||
"""The FlashInfer MXFP8 backend a 32-wide-K ue8m0 block-fp8 weight can run on."""
|
||||
"""The MXFP8 backend (FlashInfer on CUDA, gfx950 on ROCm) a 32-wide-K ue8m0
|
||||
block-fp8 weight can run on."""
|
||||
backend = get_fp8_gemm_runner_backend()
|
||||
# the Triton block kernel's ue8m0 activation quant is CUDA-only, so gfx950 takes an MXFP8 route
|
||||
if _is_hip and _is_gfx95_supported:
|
||||
if backend.is_triton():
|
||||
return Mxfp8DenseGemmBackend.GFX95_DOT_SCALED
|
||||
return Mxfp8DenseGemmBackend.GFX95_MXFP8_NATIVE
|
||||
# Explicit CUTLASS / CuTe-DSL only: they leave the weight untouched and store
|
||||
# the swizzled scale separately, so the block layout stays readable by Triton.
|
||||
if not (backend.is_flashinfer_cutedsl() or backend.is_flashinfer_cutlass()):
|
||||
@@ -758,6 +775,18 @@ def dispatch_block_fp8_mxfp8_linear(backend: Mxfp8DenseGemmBackend) -> Callable:
|
||||
return partial(
|
||||
flashinfer_mxfp8_blockscaled_linear, backend="cute-dsl", pin_tactic=True
|
||||
)
|
||||
if backend.is_gfx95_dot_scaled():
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
|
||||
dot_scaled_mxfp8_blockscaled_linear,
|
||||
)
|
||||
|
||||
return dot_scaled_mxfp8_blockscaled_linear
|
||||
if backend.is_gfx95_mxfp8_native():
|
||||
from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import (
|
||||
mxfp8_native_blockscaled_linear,
|
||||
)
|
||||
|
||||
return mxfp8_native_blockscaled_linear
|
||||
return _unsupported_mxfp8_linear
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla
|
||||
from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
__all__ = [
|
||||
@@ -31,6 +35,27 @@ def aiter_dsv3_router_gemm(
|
||||
return tgemm.mm(hidden_states, weight.detach(), otype=hidden_states.dtype)
|
||||
|
||||
|
||||
def rocm_dsv3_router_split_k(
|
||||
gate, hidden_states: torch.Tensor
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""The ROCm decode router for gate (a MoEGate): an fp32 logits buffer plus
|
||||
the split-K partials whose fixed-order sum fills it, or None when gate.forward
|
||||
applies. Only TopK.forward_cuda(..., router_logits_partials=partials) may read
|
||||
the buffer: it sums the partials into it inside the fused gate launch."""
|
||||
num_tokens = hidden_states.shape[0]
|
||||
if not 0 < num_tokens <= gate.rocm_router_max_tokens:
|
||||
return None
|
||||
if get_exec().deterministic.enable_deterministic_inference:
|
||||
return None
|
||||
partials = rocm_router_gemv_split_k(hidden_states, gate.weight)
|
||||
logits = torch.empty(
|
||||
(num_tokens, gate.weight.shape[0]),
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
return logits, partials
|
||||
|
||||
|
||||
def get_dsv3_gemm_output_zero_allocator_size(
|
||||
n_routed_experts: int, num_moe_layers: int, allocate_size: int, embedding_dim: int
|
||||
):
|
||||
|
||||
@@ -23,11 +23,12 @@ from sglang.kernels.ops.attention.dsv4.kv_layout import (
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backend
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache
|
||||
from sglang.srt.runtime_context import get_exec, get_platform, get_spec
|
||||
from sglang.srt.utils import ceil_div, is_hip
|
||||
from sglang.srt.utils import ceil_div, is_gfx95_supported, is_hip
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -170,7 +171,8 @@ def flashmla_supports_v41_kv_layouts() -> bool:
|
||||
|
||||
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
"""The (main-cache layout, compressed-cache option) for a new DeepSeek-V4
|
||||
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA."""
|
||||
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950
|
||||
AITER attention."""
|
||||
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
|
||||
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
|
||||
if mode == "v4":
|
||||
@@ -181,6 +183,17 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
and torch.version.cuda is not None
|
||||
and torch.cuda.get_device_capability()[0] == 10
|
||||
)
|
||||
if _is_hip:
|
||||
supported = (
|
||||
is_gfx95_supported() and resolve_hip_flashmla_backend() == "aiter_sparse"
|
||||
)
|
||||
if mode == "auto" and not supported:
|
||||
return KVLayout.V4, None
|
||||
if not supported:
|
||||
raise ValueError(
|
||||
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
|
||||
)
|
||||
return KVLayout.V41, option
|
||||
supported = flashmla_supports_v41_kv_layouts()
|
||||
if mode == "auto":
|
||||
if is_sm100 and supported:
|
||||
@@ -665,6 +678,21 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
loc: torch.Tensor,
|
||||
cache_k: torch.Tensor,
|
||||
) -> None:
|
||||
if self.uses_aiter_fp4_layout:
|
||||
# the FlyDSL kernels read the fp4 payload and the packed ue8m0 scales
|
||||
# from two buffers
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import (
|
||||
store_fp4_index_k_cache_split,
|
||||
)
|
||||
|
||||
return store_fp4_index_k_cache_split(
|
||||
cache_k,
|
||||
self.index_k_payload_buffer[layer_id - self.start_layer],
|
||||
self.index_k_scale_buffer[layer_id - self.start_layer],
|
||||
loc,
|
||||
page_size=self.page_size,
|
||||
rne=self.index_k_rne,
|
||||
)
|
||||
from sglang.kernels.ops.attention.dsv4.fp4_indexer import (
|
||||
store_fp4_index_k_cache,
|
||||
)
|
||||
@@ -1894,6 +1922,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
compress_layer_id
|
||||
)
|
||||
|
||||
def low_ratio_index_k_is_split(self, layer_id: int) -> bool:
|
||||
"""Whether the layer's index-K pool keeps payload and scale in the split
|
||||
FlyDSL layout (ROCm) rather than one fused [.., 68]-byte row."""
|
||||
compress_ratio, _, _ = self.layer_mapping[layer_id]
|
||||
return self._indexer_pool(compress_ratio).uses_aiter_fp4_layout
|
||||
|
||||
def get_index_k_fp4_payload_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
self.wait_layer_transfer(layer_id)
|
||||
compress_ratio, compress_layer_id, _ = self.layer_mapping[layer_id]
|
||||
@@ -1990,8 +2024,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
eps: float,
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""q ([B, H, head_dim]): rope its query heads in the same launch."""
|
||||
if self.uniform_fp8:
|
||||
assert q is None, "uniform FP8 store does not fuse query RoPE"
|
||||
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
|
||||
# then an e4m3 cast + scatter with per-tensor scale 1.0.
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import (
|
||||
@@ -2019,6 +2056,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
kvcache=self.get_swa_raw_buffer(layer_id),
|
||||
page_size=self.swa_page_size,
|
||||
layout=self.kv_layout,
|
||||
q=q,
|
||||
)
|
||||
|
||||
def set_unified_key_buffer_radix_fused_norm_rope(
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""ROCm activation route of DeepseekV2MLP: aiter's fused clamp + silu-and-mul for 128-wide half
|
||||
widths (fp8 out for a 128x128-block down_proj), else the Triton silu-and-mul-clamp, whose gfx950
|
||||
epilogue lands on down_proj's fp8 grid or emits native MXFP8. Bound by deepseek_v2 under _is_hip."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import (
|
||||
silu_and_mul_clamp_fp8_grid_supported,
|
||||
silu_and_mul_clamp_triton,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
|
||||
|
||||
def resolve_fused_clamp_route(mlp, half_width: int) -> None:
|
||||
"""Fix mlp's activation route from down_proj's loaded weight (once per layer)."""
|
||||
quant_method = mlp.down_proj.quant_method
|
||||
# the aiter kernel tiles and quantizes the half width per 128, the 128x128 block GEMM's layout
|
||||
mlp.use_fused_clamp_act_mul = half_width % 128 == 0
|
||||
mlp._fused_clamp_use_fp8 = (
|
||||
isinstance(quant_method, Fp8LinearMethod)
|
||||
and quant_method.block_quant
|
||||
and quant_method.weight_block_size == [128, 128]
|
||||
)
|
||||
# gfx950 32-block route: the activation lands on the fp8 grid, so down_proj skips its fake-quant
|
||||
mlp._hip_act_fp8_grid = bool(
|
||||
isinstance(quant_method, Fp8LinearMethod)
|
||||
and quant_method.block_fp8_as_mxfp8
|
||||
and mlp.down_proj.block_fp8_mxfp8_ready
|
||||
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
|
||||
and silu_and_mul_clamp_fp8_grid_supported(half_width)
|
||||
)
|
||||
# the native MXFP8 route takes fp8 + ue8m0 straight from the epilogue at decode token counts
|
||||
mlp._hip_act_native_consumer = bool(
|
||||
mlp._hip_act_fp8_grid
|
||||
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
|
||||
and mlp.down_proj.mxfp8_native_ready
|
||||
)
|
||||
mlp._fused_clamp_fp8_checked = True
|
||||
|
||||
|
||||
def silu_and_mul_clamp(mlp, gate_up: torch.Tensor):
|
||||
"""silu(clamp(g)) * clamp(u) for any half width (unlike the aiter kernel's
|
||||
multiple of 128), emitted on down_proj's fp8 grid or as native fp8 + ue8m0
|
||||
when the resolved route takes it."""
|
||||
return silu_and_mul_clamp_triton(
|
||||
gate_up,
|
||||
float(mlp.swiglu_limit),
|
||||
fp8_grid=mlp._hip_act_fp8_grid,
|
||||
# the native down_proj runs on fp8 + ue8m0 at every M, so hand it those directly
|
||||
emit_fp8=mlp._hip_act_native_consumer,
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
"""ROCm glue of DeepseekV2MoE: V4.1's split-K decode router and the fused all-reduce + mHC
|
||||
post of the DeepSeek-V4 decode batches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc_hip import all_reduce_mhc_post
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import (
|
||||
ALL_REDUCE_MHC_MAX_ROWS,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import _use_aiter
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if _use_aiter:
|
||||
from sglang.kernels.ops.moe.rocm_router_gate import rocm_router_max_tokens
|
||||
from sglang.srt.layers.rocm_linear_utils import rocm_dsv3_router_split_k
|
||||
|
||||
|
||||
def router_max_tokens(gate, config, is_hash_moe: bool) -> int:
|
||||
"""Rows up to which the split-K router serves gate (a MoEGate), -1 when it never does:
|
||||
V4.1 only, DSv4 keeps aiter's router GEMM and gate."""
|
||||
if not (
|
||||
_use_aiter
|
||||
and gate.is_deepseek_v4
|
||||
and getattr(config, "model_type", None) == "deepseek_v41"
|
||||
and not is_hash_moe
|
||||
):
|
||||
return -1
|
||||
return rocm_router_max_tokens(
|
||||
num_experts=config.n_routed_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
topk=config.num_experts_per_tok,
|
||||
weight_dtype=gate.weight.dtype,
|
||||
)
|
||||
|
||||
|
||||
def batch_has_images(forward_batch) -> bool:
|
||||
"""Whether a batch can carry image tokens: only extend batches with image inputs do, so
|
||||
every other batch takes the fused top-k instead of the vision one."""
|
||||
if forward_batch is None:
|
||||
return True
|
||||
return (
|
||||
forward_batch.forward_mode.is_extend() and forward_batch.contains_image_inputs()
|
||||
)
|
||||
|
||||
|
||||
def forward_gate(
|
||||
moe, hidden_states: torch.Tensor, gemm_output_zero_allocator, *, fused_gate: bool
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""The router logits of moe (a DeepseekV2MoE) and, on the split-K decode router, the
|
||||
partials moe.topk sums into them; fused_gate=False when anything else reads them."""
|
||||
if fused_gate and _use_aiter and not moe.is_hash:
|
||||
logits_and_partials = rocm_dsv3_router_split_k(moe.gate, hidden_states)
|
||||
if logits_and_partials is not None:
|
||||
return logits_and_partials
|
||||
return moe.gate(hidden_states, gemm_output_zero_allocator), None
|
||||
|
||||
|
||||
def fused_all_reduce_mhc(
|
||||
moe, mhc: Optional[MhcPostFusion], hidden_states: torch.Tensor
|
||||
) -> bool:
|
||||
"""Whether the post-experts all-reduce of moe (a DeepseekV2MoE) ran fused with the
|
||||
mHC post, for the eagerly built 1-8 row mHC states; the decoder then reads mhc.output
|
||||
and hidden_states only keeps the DP wrapper's tensor contract."""
|
||||
if not (
|
||||
mhc is not None
|
||||
and not moe._shared_expert_tp1
|
||||
and not mhc.overlap_only
|
||||
and mhc.post is not None
|
||||
and 1 <= hidden_states.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
|
||||
):
|
||||
return False
|
||||
mhc.output = all_reduce_mhc_post(
|
||||
hidden_states,
|
||||
mhc.residual,
|
||||
mhc.post,
|
||||
mhc.comb,
|
||||
get_parallel().tp_group.ca_comm,
|
||||
)
|
||||
return True
|
||||
@@ -1,11 +1,22 @@
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.layernorm.mhc_boundary_hip import HcCoefficients
|
||||
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import MhcPostFusion, use_mhc_post_fusion
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.common import is_gfx1250_supported
|
||||
|
||||
@@ -15,6 +26,22 @@ _is_hip = is_hip()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
_is_gfx1250_supported = is_gfx1250_supported()
|
||||
|
||||
if _is_hip:
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc_hip import (
|
||||
all_reduce_mhc_post,
|
||||
)
|
||||
from sglang.kernels.ops.layernorm.mhc_boundary_hip import (
|
||||
hc_boundary_fused_deferred,
|
||||
)
|
||||
if _is_gfx95_supported:
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_gfx95_dense import (
|
||||
post_attention_norm,
|
||||
)
|
||||
|
||||
# the fused all-reduce + hc_post kernel: DeepSeek-V4.1's hidden size, up to this many rows
|
||||
MHC_HIDDEN_SIZE = 5120
|
||||
ALL_REDUCE_MHC_MAX_ROWS = 8
|
||||
|
||||
_FUSED_HC_POST_PRE_M_THRESHOLD = 64
|
||||
_FUSED_HC_POST_PRE_CACHE: dict[tuple, dict[str, torch.Tensor]] = {}
|
||||
_TRITON_MHC_POST_PRE_OPS = None
|
||||
@@ -421,3 +448,178 @@ def apply_mhc_post_pre_boundary(
|
||||
)
|
||||
post_out = post_out.squeeze(-1) if post_out.ndim == 3 else post_out
|
||||
return residual, layer_input_out, post_out, comb_out, True
|
||||
|
||||
|
||||
def hc_boundary(
|
||||
layer,
|
||||
x: Optional[torch.Tensor],
|
||||
residual: torch.Tensor,
|
||||
post: Optional[torch.Tensor],
|
||||
comb: Optional[torch.Tensor],
|
||||
pre_prev: Optional[torch.Tensor],
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor, "HcCoefficients"]:
|
||||
"""Fused sublayer boundary (ROCm): apply the pending hc_post of x onto residual, collapse
|
||||
with pre_prev (copy 0 when None) and take the mixing statistics. Returns (new_residual, y,
|
||||
coefficients); the coefficients' reduce + sinkhorn is still pending (see HcCoefficients)."""
|
||||
new_residual, y, coefficients = hc_boundary_fused_deferred(
|
||||
x,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
pre_prev,
|
||||
hc_fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
layer.hc_mult,
|
||||
layer.hc_sinkhorn_iters,
|
||||
layer.rms_norm_eps,
|
||||
layer.hc_eps,
|
||||
)
|
||||
if new_residual is None:
|
||||
new_residual = residual
|
||||
if y is None:
|
||||
y = new_residual[:, 0, :].contiguous()
|
||||
return new_residual, y, coefficients
|
||||
|
||||
|
||||
def _can_fuse_mhc(layer, residual: torch.Tensor, forward_batch) -> bool:
|
||||
return (
|
||||
_is_gfx95_supported
|
||||
and 1 <= residual.shape[0] <= ALL_REDUCE_MHC_MAX_ROWS
|
||||
and residual.shape[1:] == (4, MHC_HIDDEN_SIZE)
|
||||
and residual.dtype == torch.bfloat16
|
||||
and residual.is_contiguous()
|
||||
and layer.config.model_type == "deepseek_v41"
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
and get_parallel().attn_dp_size == 1
|
||||
and get_parallel().tp_size == 4
|
||||
and not layer.dsa_enable_prefill_cp
|
||||
and not get_forward().sp_active
|
||||
and not is_batch_invariant_mode_enabled()
|
||||
and not get_exec().deterministic.enable_deterministic_inference
|
||||
)
|
||||
|
||||
|
||||
def _make_mhc_fusion(
|
||||
residual: torch.Tensor, coefficients: "HcCoefficients", comm
|
||||
) -> Optional[MhcPostFusion]:
|
||||
# comm is the group's aiter CustomAllreduce, or None when custom all-reduce is off
|
||||
if comm is None or comm.disabled or not comm.enable_register_for_capturing:
|
||||
return None
|
||||
coefficients.materialize()
|
||||
return MhcPostFusion(
|
||||
residual, coefficients.post, coefficients.comb, None, pre=coefficients.pre
|
||||
)
|
||||
|
||||
|
||||
def attention_mhc_fusion(layer, residual, coefficients, forward_batch):
|
||||
if not (
|
||||
_can_fuse_mhc(layer, residual, forward_batch)
|
||||
and layer.self_attn.attn_tp_size == 4
|
||||
and layer.self_attn.wo_b.reduce_results
|
||||
):
|
||||
return None
|
||||
return _make_mhc_fusion(
|
||||
residual, coefficients, get_parallel().attn_tp_group.ca_comm
|
||||
)
|
||||
|
||||
|
||||
def moe_mhc_fusion(layer, residual, coefficients, forward_batch):
|
||||
if not (
|
||||
_can_fuse_mhc(layer, residual, forward_batch)
|
||||
and layer.mlp.tp_size == 4
|
||||
and not layer.mlp._shared_expert_tp1
|
||||
and get_moe_a2a_backend().is_none()
|
||||
):
|
||||
return None
|
||||
return _make_mhc_fusion(residual, coefficients, get_parallel().tp_group.ca_comm)
|
||||
|
||||
|
||||
def apply_attention_mhc(x: torch.Tensor, state: MhcPostFusion) -> None:
|
||||
# a lazily recorded state still needs its stats before the fused kernel reads them
|
||||
state.materialize_stats()
|
||||
if state.stats_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(state.stats_stream)
|
||||
state.output = all_reduce_mhc_post(
|
||||
x, state.residual, state.post, state.comb, get_parallel().attn_tp_group.ca_comm
|
||||
)
|
||||
|
||||
|
||||
def forward_hc_pre_from_prev_fused_boundary(
|
||||
layer,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
pending_post: Optional[Tuple[torch.Tensor, ...]],
|
||||
defer_post: bool,
|
||||
) -> Tuple[Optional[torch.Tensor], torch.Tensor, Optional[Tuple[torch.Tensor, ...]]]:
|
||||
"""ROCm form of DeepseekV4DecoderLayer.forward_hc_pre_from_prev.
|
||||
pending_post is the previous layer's unapplied FFN hc_post (x, residual, post,
|
||||
comb); with defer_post this layer's is returned the same way and hidden_states is None."""
|
||||
if pending_post is None:
|
||||
pending_post = (None, hidden_states, None, None)
|
||||
residual, x, attn_coefficients = hc_boundary(
|
||||
layer,
|
||||
*pending_post,
|
||||
prev_pre,
|
||||
layer.hc_attn_fn,
|
||||
layer.hc_attn_scale,
|
||||
layer.hc_attn_base,
|
||||
)
|
||||
# the boundary's reduce + sinkhorn rides in the norm launch
|
||||
x, x_quant = layer._input_norm(
|
||||
x, allow_aiter_quant=False, coefficients=attn_coefficients
|
||||
)
|
||||
with layer.self_attn.maybe_use_decode_attn_tp(forward_batch):
|
||||
mhc = attention_mhc_fusion(layer, residual, attn_coefficients, forward_batch)
|
||||
with use_mhc_post_fusion(mhc):
|
||||
x = layer.self_attn(
|
||||
x=x, positions=positions, forward_batch=forward_batch, x_quant=x_quant
|
||||
)
|
||||
if mhc is not None:
|
||||
residual = mhc.output
|
||||
x = None
|
||||
residual, x, ffn_coefficients = hc_boundary(
|
||||
layer,
|
||||
x,
|
||||
residual,
|
||||
attn_coefficients.post if mhc is None else None,
|
||||
attn_coefficients.comb if mhc is None else None,
|
||||
attn_coefficients.pre,
|
||||
layer.hc_ffn_fn,
|
||||
layer.hc_ffn_scale,
|
||||
layer.hc_ffn_base,
|
||||
)
|
||||
x = _gfx95_dense_post_attention_norm(layer, x, ffn_coefficients)
|
||||
mhc = moe_mhc_fusion(layer, residual, ffn_coefficients, forward_batch)
|
||||
with use_mhc_post_fusion(mhc):
|
||||
x = layer._run_moe_ffn_dp_sync(
|
||||
x, forward_batch, input_ids=input_ids, input_ids_global=input_ids_global
|
||||
)
|
||||
ffn_pre, ffn_post, ffn_comb = ffn_coefficients.tensors()
|
||||
if mhc is not None and mhc.output is not None:
|
||||
# Reduction already applied post. The next boundary consumes this
|
||||
# materialized residual, including when it would normally defer post.
|
||||
return mhc.output, ffn_pre, None
|
||||
if defer_post:
|
||||
return None, ffn_pre, (x, residual, ffn_post, ffn_comb)
|
||||
return layer.hc_post(x, residual, ffn_post, ffn_comb), ffn_pre, None
|
||||
|
||||
|
||||
def _gfx95_dense_post_attention_norm(layer, x: torch.Tensor, coefficients):
|
||||
"""layer.post_attention_layernorm(x) hosting the pending reduce + sinkhorn where
|
||||
the gfx950 norm kernel is available; the module norm plus a standalone reduce +
|
||||
sinkhorn elsewhere."""
|
||||
if _is_gfx95_supported:
|
||||
return post_attention_norm(layer, x, coefficients)
|
||||
coefficients.materialize()
|
||||
return layer.post_attention_layernorm(x)
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""gfx950 dense route of the DeepSeek-V4 attention for 32-wide-block (V4.1) fp8 checkpoints: the
|
||||
fused RMSNorm + fake-quant producers hand wqkv_a / wq_b their operand on the fp8 grid
|
||||
(Fp8GridActivation) or as native MXFP8 (Mxfp8Activation); deepseek_v4 binds it under _is_hip."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn
|
||||
from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import (
|
||||
Fp8GridActivation,
|
||||
Mxfp8Activation,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import (
|
||||
rmsnorm_fake_quant_fp8,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from sglang.srt.layers.quantization.fp8_utils import resolve_block_fp8_mxfp8_backend
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.common import is_gfx1250_supported
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_hip = is_hip()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
_is_gfx1250_supported = is_gfx1250_supported()
|
||||
_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip
|
||||
|
||||
# V4.1's wo_a route on gfx950: an aiter batched GEMM fork with wo_b's fp8-grid rounding in its
|
||||
# epilogue; None keeps the aiter kernel
|
||||
_wo_a_fp8_grid_gemm = None
|
||||
if _use_aiter and _is_gfx95_supported:
|
||||
from sglang.kernels.ops.gemm.gfx95_batched_gemm_bf16_fp8_grid import (
|
||||
batched_gemm_bf16_fp8_grid as _wo_a_fp8_grid_gemm,
|
||||
)
|
||||
|
||||
|
||||
def fused_rmsnorm_fp8_quant_eligible(
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
) -> bool:
|
||||
"""Whether the aiter fused RMSNorm + fp8 quant applies: its (fp8, 128-group scale)
|
||||
output is consumed only by the 128x128-block dense GEMM."""
|
||||
return bool(
|
||||
_use_aiter
|
||||
and (_is_gfx95_supported or _is_gfx1250_supported)
|
||||
and isinstance(quant_config, Fp8Config)
|
||||
and quant_config.weight_block_size == [128, 128]
|
||||
)
|
||||
|
||||
|
||||
def fused_rmsnorm_fake_quant_eligible(
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
) -> bool:
|
||||
"""Whether rmsnorm_fake_quant_fp8 applies: gfx950 with a 32-wide-block checkpoint
|
||||
(V4.1), whose dense route takes the norm output already on the fp8 grid as an
|
||||
Fp8GridActivation."""
|
||||
if not (_is_hip and _is_gfx95_supported and isinstance(quant_config, Fp8Config)):
|
||||
return False
|
||||
block = quant_config.weight_block_size
|
||||
return (
|
||||
block is not None
|
||||
and block[1] == 32
|
||||
and quant_config.scale_fmt == "ue8m0"
|
||||
and resolve_block_fp8_mxfp8_backend().is_gfx95_mxfp8_native()
|
||||
)
|
||||
|
||||
|
||||
def _native_mxfp8_consumer(linear: Optional[nn.Module]) -> bool:
|
||||
"""Whether linear runs the gfx950 native MXFP8 route, which consumes fp8 + ue8m0
|
||||
scales directly at every M."""
|
||||
if linear is None:
|
||||
return False
|
||||
quant_method = linear.quant_method
|
||||
return (
|
||||
isinstance(quant_method, Fp8LinearMethod)
|
||||
and quant_method.block_fp8_as_mxfp8
|
||||
and linear.block_fp8_mxfp8_ready
|
||||
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
|
||||
and linear.mxfp8_native_ready
|
||||
)
|
||||
|
||||
|
||||
def _fake_quant_applies(norm: nn.Module, x: torch.Tensor) -> bool:
|
||||
return x.dim() == 2 and x.dtype == norm.weight.dtype
|
||||
|
||||
|
||||
def q_norm_fake_quant(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]:
|
||||
"""attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b
|
||||
consumes, already on the fp8 grid); the plain norm, twice, when the rows are not
|
||||
a 2-D bf16 batch."""
|
||||
if not _fake_quant_applies(attn.q_norm, q_lora):
|
||||
q_lora = attn.q_norm(q_lora)
|
||||
return q_lora, q_lora
|
||||
if not attn._wq_b_native_consumer_checked:
|
||||
attn._wq_b_native_consumer = _native_mxfp8_consumer(attn.wq_b)
|
||||
attn._wq_b_native_consumer_checked = True
|
||||
q_for_wq_b, q_lora = rmsnorm_fake_quant_fp8(
|
||||
q_lora,
|
||||
attn.q_norm.weight.data,
|
||||
attn.q_norm.variance_epsilon,
|
||||
emit_fp8=attn._wq_b_native_consumer,
|
||||
)
|
||||
return q_lora, q_for_wq_b
|
||||
|
||||
|
||||
def input_norm_fake_quant(
|
||||
layer, hidden_states: torch.Tensor, coefficients=None
|
||||
) -> Tuple[torch.Tensor, Optional[object]]:
|
||||
"""layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the fp8-grid operand
|
||||
of its dense projections, or None for non-2-D / non-bf16 rows). coefficients rides in the norm
|
||||
launch when the fused kernel runs, else it is materialized here."""
|
||||
norm = layer.input_layernorm
|
||||
if not _fake_quant_applies(norm, hidden_states):
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
return norm(hidden_states), None
|
||||
if not layer._wqkv_a_native_consumer_checked:
|
||||
# wqkv_a exists only when the q / kv projections are fused
|
||||
layer._wqkv_a_native_consumer = _native_mxfp8_consumer(
|
||||
layer.self_attn.wqkv_a if layer.self_attn.fuse_wqa_wkv else None
|
||||
)
|
||||
layer._wqkv_a_native_consumer_checked = True
|
||||
emit_fp8 = layer._wqkv_a_native_consumer
|
||||
if coefficients is not None and not coefficients.materialized:
|
||||
x_quant, hidden_states = rmsnorm_with_sinkhorn(
|
||||
hidden_states,
|
||||
norm.weight.data,
|
||||
norm.variance_epsilon,
|
||||
coefficients,
|
||||
emit_fp8=emit_fp8,
|
||||
)
|
||||
return hidden_states, x_quant
|
||||
x_quant, hidden_states = rmsnorm_fake_quant_fp8(
|
||||
hidden_states,
|
||||
norm.weight.data,
|
||||
norm.variance_epsilon,
|
||||
emit_fp8=emit_fp8,
|
||||
)
|
||||
return hidden_states, x_quant
|
||||
|
||||
|
||||
def post_attention_norm(layer, x: torch.Tensor, coefficients=None) -> torch.Tensor:
|
||||
"""layer.post_attention_layernorm(x); with coefficients (HcCoefficients of the
|
||||
boundary that produced x) still pending, the reduce + sinkhorn rides in the norm
|
||||
launch, which then is the Triton row norm rather than the aiter one."""
|
||||
norm = layer.post_attention_layernorm
|
||||
if (
|
||||
coefficients is None
|
||||
or coefficients.materialized
|
||||
or not _fake_quant_applies(norm, x)
|
||||
):
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
return norm(x)
|
||||
_, out = rmsnorm_with_sinkhorn(
|
||||
x, norm.weight.data, norm.variance_epsilon, coefficients, fake_quant=False
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def live_rows(activation, num_tokens: int):
|
||||
"""The first num_tokens rows of a break input; the gfx950 fused q_norm
|
||||
hands the indexer its Fp8GridActivation wrapper, whose rows live in .x."""
|
||||
if isinstance(activation, Fp8GridActivation):
|
||||
return Fp8GridActivation(activation.x[:num_tokens])
|
||||
if isinstance(activation, Mxfp8Activation):
|
||||
return Mxfp8Activation(activation.q[:num_tokens], activation.scale[:num_tokens])
|
||||
return activation[:num_tokens]
|
||||
|
||||
|
||||
def wo_b_takes_fp8_grid(attn) -> bool:
|
||||
"""Whether attn.wo_b consumes an Fp8GridActivation, which the wo_a GEMM
|
||||
then emits from its epilogue; resolved on first use, once the weights are loaded."""
|
||||
if attn._wo_b_fp8_grid_operand is None:
|
||||
quant_method = attn.wo_b.quant_method
|
||||
attn._wo_b_fp8_grid_operand = bool(
|
||||
_wo_a_fp8_grid_gemm is not None
|
||||
and isinstance(quant_method, Fp8LinearMethod)
|
||||
and quant_method.block_fp8_as_mxfp8
|
||||
and attn.wo_b.block_fp8_mxfp8_ready
|
||||
and quant_method.mxfp8_dense_backend.is_gfx95_mxfp8_native()
|
||||
)
|
||||
return attn._wo_b_fp8_grid_operand
|
||||
|
||||
|
||||
def wo_a_fp8_grid_matmul(o: torch.Tensor, wo_a: torch.Tensor, fp8_grid: bool):
|
||||
"""o [T, G, D] @ wo_a [G, R, D]^T through the gfx950 fork of the aiter batched
|
||||
GEMM: bf16 [T, G, R], or with fp8_grid rounded onto wo_b's fp8 grid as
|
||||
an Fp8GridActivation [T, G * R]; None when the fork did not import."""
|
||||
if _wo_a_fp8_grid_gemm is None:
|
||||
return None
|
||||
# the split-K regime ends at 64 rows, so a request's verify and decode rows would
|
||||
# take different reduction orders; deterministic inference keeps the single chain
|
||||
split_k = False if get_exec().deterministic.enable_deterministic_inference else None
|
||||
y = _wo_a_fp8_grid_gemm(o, wo_a, fp8_grid=fp8_grid, split_k=split_k)
|
||||
if fp8_grid:
|
||||
return Fp8GridActivation(y)
|
||||
return y.view(o.shape[0], wo_a.shape[0], wo_a.shape[1])
|
||||
@@ -0,0 +1,247 @@
|
||||
"""ROCm glue of deepseek_v4: the branches the model takes under _is_hip, bound there as
|
||||
_hip. The gfx950 dense fp8 routes live in deepseek_v4_gfx95_dense, the fused mHC
|
||||
boundary in deepseek_v4_fused_mhc."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h
|
||||
from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.hip_flash_mla import (
|
||||
hip_attention_needs_head_pad,
|
||||
resolve_hip_flashmla_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v4_gfx95_dense as gfx95_dense
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_fused_mhc import ( # noqa: F401 deepseek_v4 reaches apply_attention_mhc through this module
|
||||
apply_attention_mhc,
|
||||
forward_hc_pre_from_prev_fused_boundary,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_gfx95_supported
|
||||
|
||||
live_rows = gfx95_dense.live_rows
|
||||
wo_a_fp8_grid_matmul = gfx95_dense.wo_a_fp8_grid_matmul
|
||||
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
|
||||
|
||||
# ---- MqaAttentionBase / MQALayer ----
|
||||
|
||||
|
||||
def init_mqa_layer(attn, quant_config) -> None:
|
||||
"""The ROCm state of MQALayer.__init__. On the gfx950 32-block route q_norm also emits
|
||||
the fp8-grid operand of wq_b; whether wq_b consumes native MXFP8, and wo_b the fp8-grid
|
||||
operand wo_a emits, resolve on first use, once the weights are loaded."""
|
||||
attn.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible(
|
||||
quant_config
|
||||
)
|
||||
attn._wq_b_native_consumer_checked = False
|
||||
attn._wq_b_native_consumer = False
|
||||
attn._wo_b_fp8_grid_operand = None
|
||||
|
||||
|
||||
def wo_a_emits_fp8_grid(attn) -> bool:
|
||||
"""Whether V4.1's gfx950 wo_a GEMM rounds its output onto wo_b's fp8 grid."""
|
||||
return attn.is_dsv41 and gfx95_dense.wo_b_takes_fp8_grid(attn)
|
||||
|
||||
|
||||
def wo_a_split_k_allowed() -> bool:
|
||||
"""Whether the V4.1 gfx950 wo_a decode / verify kernels may run: their split-K
|
||||
reduction order depends on the row count, which batch-invariant and deterministic
|
||||
inference rule out."""
|
||||
return not (
|
||||
is_batch_invariant_mode_enabled()
|
||||
or get_exec().deterministic.enable_deterministic_inference
|
||||
)
|
||||
|
||||
|
||||
def use_fused_qk_norm_rope(attn) -> bool:
|
||||
return bool(envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get() and attn.q_head_norm)
|
||||
|
||||
|
||||
def q_norm_for_wq_b(attn, q_lora: torch.Tensor) -> Tuple[torch.Tensor, object]:
|
||||
"""attn.q_norm(q_lora) as (the bf16 norm the indexer reads, the operand wq_b
|
||||
consumes); on the gfx950 32-block route the second is already on the fp8 grid."""
|
||||
if attn.fused_rmsnorm_fake_quant:
|
||||
return gfx95_dense.q_norm_fake_quant(attn, q_lora)
|
||||
q_lora = attn.q_norm(q_lora)
|
||||
return q_lora, q_lora
|
||||
|
||||
|
||||
def fuses_q_rope_into_k_store(
|
||||
attn, q_out: Optional[torch.Tensor], *, unified: bool, use_cp: bool
|
||||
) -> bool:
|
||||
"""Whether the K norm-rope-store launch ropes the query heads too: only the plain store
|
||||
path, where it is the cache writer and the query is consumed in place (no padded copy)."""
|
||||
return (
|
||||
not unified
|
||||
and not use_cp
|
||||
and q_out is None
|
||||
and not attn.q_head_norm
|
||||
and not envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get()
|
||||
)
|
||||
|
||||
|
||||
def wq_b_unroped(attn, q) -> torch.Tensor:
|
||||
"""attn._compute_q_b without the RoPE, which the K store launch applies."""
|
||||
q, _ = attn.wq_b(q)
|
||||
return q.view(-1, attn.n_local_heads, attn.head_dim)
|
||||
|
||||
|
||||
def compute_q_b(
|
||||
attn,
|
||||
q_lora: torch.Tensor,
|
||||
q_for_wqb,
|
||||
positions: torch.Tensor,
|
||||
q_out: Optional[torch.Tensor],
|
||||
unified: bool,
|
||||
use_cp: bool,
|
||||
):
|
||||
"""attn._compute_q_b(q_for_wqb, positions, q_out) as (q, the q_lora the indexer reads,
|
||||
whether the K norm-rope-store launch ropes q). The indexer takes the fused operand:
|
||||
its wq_b would re-round onto the same grid."""
|
||||
fuse_q_rope = _is_gfx95_supported and fuses_q_rope_into_k_store(
|
||||
attn, q_out, unified=unified, use_cp=use_cp
|
||||
)
|
||||
if fuse_q_rope:
|
||||
q = wq_b_unroped(attn, q_for_wqb)
|
||||
else:
|
||||
q = attn._compute_q_b(q_for_wqb, positions, q_out)
|
||||
return q, q_for_wqb, fuse_q_rope
|
||||
|
||||
|
||||
def skip_head_pad(attn) -> bool:
|
||||
# only tilelang is built for padded head widths; aiter and Triton take the real head count
|
||||
return attn.attn_tp_size > 1 and not hip_attention_needs_head_pad()
|
||||
|
||||
|
||||
def attention_inv_rope(
|
||||
attn,
|
||||
positions: torch.Tensor,
|
||||
forward_batch,
|
||||
unified: bool,
|
||||
*,
|
||||
wo_a_applies_inv_rope: bool,
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""(freqs_real, positions) for aiter's gfx950 sparse kernel to apply the inverse RoPE
|
||||
itself; None for the other kernels, and where the fp8 wo_a front end or the prefill
|
||||
graph op keeps it."""
|
||||
if (
|
||||
unified
|
||||
or not _is_gfx95_supported
|
||||
or resolve_hip_flashmla_backend() != "aiter_sparse"
|
||||
or wo_a_applies_inv_rope
|
||||
):
|
||||
return None
|
||||
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
|
||||
return None
|
||||
return torch.view_as_real(attn.freqs_cis).flatten(-2), positions
|
||||
|
||||
|
||||
# ---- DeepseekV4DecoderLayer ----
|
||||
|
||||
|
||||
def init_decoder_layer(layer, quant_config) -> None:
|
||||
"""The ROCm state of DeepseekV4DecoderLayer.__init__. input_layernorm also emits the
|
||||
pre-quantized operand of the dense projections (whether wqkv_a consumes native MXFP8
|
||||
resolves on first use), and on gfx950 one launch runs hc_post, the pre-collapse and
|
||||
the mixing stats; the kernel only supports hc_mult 4."""
|
||||
layer.fused_rmsnorm_fp8_quant = gfx95_dense.fused_rmsnorm_fp8_quant_eligible(
|
||||
quant_config
|
||||
)
|
||||
layer.fused_rmsnorm_fake_quant = gfx95_dense.fused_rmsnorm_fake_quant_eligible(
|
||||
quant_config
|
||||
)
|
||||
layer._wqkv_a_native_consumer_checked = False
|
||||
layer._wqkv_a_native_consumer = False
|
||||
layer.hc_boundary_fused = (
|
||||
_is_gfx95_supported and layer.hc_pre_from_prev_sublayer and layer.hc_mult == 4
|
||||
)
|
||||
|
||||
|
||||
def hc_post(layer, x, residual, post, comb) -> Optional[torch.Tensor]:
|
||||
"""V4.1's gfx950 hc_post at 768-4096 rows: the split-H kernel with 2048-wide blocks,
|
||||
whose ordinary stores preserve locality for the following mHC reader; None elsewhere."""
|
||||
if not (
|
||||
_is_gfx95_supported
|
||||
and layer.config.model_type == "deepseek_v41"
|
||||
and 768 <= x.shape[0] <= 4096
|
||||
and x.shape[1] == 5120
|
||||
and residual.shape == (x.shape[0], 4, 5120)
|
||||
and post.shape == (x.shape[0], 4)
|
||||
and comb.shape == (x.shape[0], 4, 4)
|
||||
and x.dtype == residual.dtype == torch.bfloat16
|
||||
and post.dtype == comb.dtype == torch.float32
|
||||
and all(t.is_contiguous() for t in (x, residual, post, comb))
|
||||
):
|
||||
return None
|
||||
return mhc_post_split_h(x, residual, post, comb, block_size=2048)
|
||||
|
||||
|
||||
def input_norm(
|
||||
layer,
|
||||
hidden_states: torch.Tensor,
|
||||
allow_aiter_quant: bool,
|
||||
coefficients,
|
||||
fused_rmsnorm_fp8_quant,
|
||||
) -> Tuple[torch.Tensor, Optional[object]]:
|
||||
"""layer.input_layernorm(hidden_states) as (the bf16 norm attention reads, the
|
||||
pre-quantized operand of its dense projections or None). coefficients is the fused
|
||||
boundary's pending reduce + sinkhorn, hosted by the gfx950 norm launch when that one runs."""
|
||||
if layer.fused_rmsnorm_fp8_quant and allow_aiter_quant:
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
x_quant, hidden_states = fused_rmsnorm_fp8_quant(
|
||||
hidden_states, layer.input_layernorm.weight, layer.rms_norm_eps
|
||||
)
|
||||
return hidden_states, x_quant
|
||||
if layer.fused_rmsnorm_fake_quant:
|
||||
return gfx95_dense.input_norm_fake_quant(layer, hidden_states, coefficients)
|
||||
if coefficients is not None:
|
||||
coefficients.materialize()
|
||||
return layer.input_layernorm(hidden_states), None
|
||||
|
||||
|
||||
# ---- DeepseekV4Model layer loop ----
|
||||
|
||||
|
||||
def forward_layer_fused_boundary(
|
||||
model,
|
||||
i: int,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: Optional[torch.Tensor],
|
||||
input_ids: torch.Tensor,
|
||||
forward_batch,
|
||||
input_ids_global: torch.Tensor,
|
||||
prev_pre: Optional[torch.Tensor],
|
||||
pending_post: Optional[Tuple[torch.Tensor, ...]],
|
||||
capture_dspark: bool,
|
||||
):
|
||||
"""Layer i through the fused mHC boundary. Its FFN hc_post stays pending for the next
|
||||
layer's boundary launch unless that layer reads the residual stream first (Engram gate,
|
||||
DSpark capture) or there is none; hidden_states is None while a post is pending."""
|
||||
nxt = i + 1
|
||||
defer_post = (
|
||||
nxt < model.end_layer
|
||||
and model.layers[nxt].engram is None
|
||||
and not (capture_dspark and nxt in model.dspark_layers_to_capture)
|
||||
)
|
||||
return forward_hc_pre_from_prev_fused_boundary(
|
||||
model.layers[i],
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
prev_pre=prev_pre,
|
||||
pending_post=pending_post,
|
||||
defer_post=defer_post,
|
||||
)
|
||||
@@ -225,6 +225,13 @@ if _use_aiter_gfx95:
|
||||
if _use_aiter:
|
||||
pass
|
||||
|
||||
if _is_hip:
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_act as _hip_act
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v2_hip_moe as _hip_moe
|
||||
else:
|
||||
_hip_act = None
|
||||
_hip_moe = None
|
||||
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16
|
||||
elif _is_npu:
|
||||
@@ -401,20 +408,14 @@ class DeepseekV2MLP(nn.Module):
|
||||
)
|
||||
return down_output
|
||||
|
||||
if self.use_fused_clamp_act_mul and not self._fused_clamp_fp8_checked:
|
||||
_hip_act.resolve_fused_clamp_route(self, gate_up.shape[-1] // 2)
|
||||
|
||||
if self.use_fused_clamp_act_mul and self.swiglu_limit is not None:
|
||||
from aiter.ops.triton.fusions.fused_clamp_act_mul import (
|
||||
fused_clamp_act_mul,
|
||||
)
|
||||
|
||||
if not self._fused_clamp_fp8_checked:
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod
|
||||
|
||||
qm = getattr(self.down_proj, "quant_method", None)
|
||||
self._fused_clamp_use_fp8 = (
|
||||
isinstance(qm, Fp8LinearMethod) and qm.block_quant
|
||||
)
|
||||
self._fused_clamp_fp8_checked = True
|
||||
|
||||
if self._fused_clamp_use_fp8:
|
||||
from aiter import dtypes
|
||||
|
||||
@@ -442,7 +443,9 @@ class DeepseekV2MLP(nn.Module):
|
||||
|
||||
# Fallback: fused silu+clamp kernel (still faster than unfused)
|
||||
elif self.swiglu_limit is not None:
|
||||
if _is_npu:
|
||||
if _is_hip:
|
||||
x = _hip_act.silu_and_mul_clamp(self, gate_up)
|
||||
elif _is_npu:
|
||||
x = torch.ops.npu.npu_clipped_swiglu(
|
||||
gate_up,
|
||||
alpha=1,
|
||||
@@ -514,6 +517,10 @@ class MoEGate(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
weight_dtype=self.weight.dtype,
|
||||
)
|
||||
# Rows up to which the ROCm split-K router serves the gate (-1: never)
|
||||
self.rocm_router_max_tokens = (
|
||||
_hip_moe.router_max_tokens(self, config, is_hash_moe) if _is_hip else -1
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -939,6 +946,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
|
||||
if use_vision_topk and _is_hip:
|
||||
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
|
||||
if not self._enable_a2a_moe:
|
||||
if self._can_dual_stream_graph(hidden_states):
|
||||
fwd = get_forward()
|
||||
@@ -961,6 +971,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
)
|
||||
else:
|
||||
return self.forward_normal(
|
||||
@@ -970,6 +981,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global=input_ids_global,
|
||||
skip_shared_experts=skip_shared_experts,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
)
|
||||
else:
|
||||
return self.forward_deepep(
|
||||
@@ -983,6 +995,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Note(kpham-sgl): issue order satisfies 3 constraints:
|
||||
# - no stream explosion: main (routed) issued before alt block -> capture reuses 1 alt stream;
|
||||
@@ -1016,7 +1029,16 @@ class DeepseekV2MoE(nn.Module):
|
||||
)
|
||||
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
if _is_hip:
|
||||
router_logits, router_logits_partials = _hip_moe.forward_gate(
|
||||
self,
|
||||
hidden_states,
|
||||
gemm_output_zero_allocator,
|
||||
fused_gate=not use_flashinfer_trtllm_bypass and not use_vision_topk,
|
||||
)
|
||||
else:
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
router_logits_partials = None
|
||||
if use_flashinfer_trtllm_bypass:
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
@@ -1029,7 +1051,9 @@ class DeepseekV2MoE(nn.Module):
|
||||
if getattr(self, "is_hash", False)
|
||||
else {}
|
||||
)
|
||||
if self.gate.e_score_correction_bias_vl is not None:
|
||||
if router_logits_partials is not None:
|
||||
topk_kwargs["router_logits_partials"] = router_logits_partials
|
||||
if use_vision_topk:
|
||||
topk_output = vision_topk(
|
||||
self,
|
||||
router_logits,
|
||||
@@ -1226,6 +1250,10 @@ class DeepseekV2MoE(nn.Module):
|
||||
)
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
@@ -1243,6 +1271,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
skip_shared_experts: bool = False,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if hasattr(self, "shared_experts") and use_intel_amx_backend(
|
||||
self.shared_experts.gate_up_proj
|
||||
@@ -1277,13 +1306,24 @@ class DeepseekV2MoE(nn.Module):
|
||||
pre_quant_input=pre_quant_input,
|
||||
)
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
if _is_hip:
|
||||
router_logits, router_logits_partials = _hip_moe.forward_gate(
|
||||
self,
|
||||
hidden_states,
|
||||
gemm_output_zero_allocator,
|
||||
fused_gate=not use_vision_topk,
|
||||
)
|
||||
else:
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
router_logits_partials = None
|
||||
topk_kwargs = (
|
||||
{"input_ids": input_ids_global}
|
||||
if getattr(self, "is_hash", False)
|
||||
else {}
|
||||
)
|
||||
if self.gate.e_score_correction_bias_vl is not None:
|
||||
if router_logits_partials is not None:
|
||||
topk_kwargs["router_logits_partials"] = router_logits_partials
|
||||
if use_vision_topk:
|
||||
topk_output = vision_topk(
|
||||
self,
|
||||
router_logits,
|
||||
@@ -1382,6 +1422,10 @@ class DeepseekV2MoE(nn.Module):
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
if _is_hip and _hip_moe.fused_all_reduce_mhc(
|
||||
self, mhc, final_hidden_states
|
||||
):
|
||||
return final_hidden_states
|
||||
if mhc is not None:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
final_hidden_states = post_experts_all_reduce(final_hidden_states)
|
||||
|
||||
@@ -425,7 +425,9 @@ _wo_a_aiter_batched_gemm_disabled = False
|
||||
_wo_a_fp8_mxscale = None
|
||||
_wo_a_fp8_mxscale_fused_invrope = None
|
||||
_wo_a_weight_scale_to_e8m0 = None
|
||||
_hip = None
|
||||
if _is_hip:
|
||||
from sglang.srt.models.deepseek_common.amd import deepseek_v4_hip as _hip
|
||||
from sglang.srt.models.deepseek_common.amd.deepseek_v4_wo_a_fp8 import (
|
||||
apply_wo_a_fp8_mxscale,
|
||||
apply_wo_a_fp8_mxscale_fused_invrope,
|
||||
@@ -459,31 +461,48 @@ def _apply_wo_a_bf16_matmul(
|
||||
fuse_mxfp8_quant: bool = False,
|
||||
is_prefill: bool = False,
|
||||
fast_path: bool = False,
|
||||
fp8_grid: bool = False,
|
||||
) -> torch.Tensor | Mxfp8SwizzledInput:
|
||||
# o [T, G, D] @ wo_a [G, R, D] -> [T, G, R]; the fast paths below are gated
|
||||
# on the exact validated TP4 shapes and write token-major output directly.
|
||||
global _wo_a_aiter_batched_gemm_disabled
|
||||
if (
|
||||
fast_path
|
||||
and _is_cuda
|
||||
# the gfx950 routes, like the CUDA ones, are V4.1's (fast_path); DSv4 keeps the
|
||||
# aiter batched GEMM / einsum below
|
||||
hip_fast_path = fast_path and _is_hip and _is_gfx95_supported
|
||||
hip_decode_verify = (
|
||||
hip_fast_path
|
||||
and (
|
||||
(
|
||||
is_decode
|
||||
and o.shape[0] == 1
|
||||
and (get_platform().is_blackwell or get_platform().is_sm90)
|
||||
)
|
||||
or (
|
||||
is_target_verify
|
||||
and 0 < o.shape[0] <= 384
|
||||
and get_platform().is_blackwell
|
||||
)
|
||||
or (
|
||||
is_prefill
|
||||
and 4096 <= o.shape[0] <= 65536
|
||||
and get_platform().is_blackwell
|
||||
(is_decode and o.shape[0] == 1 and o.is_contiguous())
|
||||
or (is_target_verify and 2 <= o.shape[0] <= 384)
|
||||
)
|
||||
and _hip.wo_a_split_k_allowed()
|
||||
)
|
||||
if (
|
||||
(
|
||||
fast_path
|
||||
and _is_cuda
|
||||
and (
|
||||
(
|
||||
is_decode
|
||||
and o.shape[0] == 1
|
||||
and (get_platform().is_blackwell or get_platform().is_sm90)
|
||||
)
|
||||
or (
|
||||
is_target_verify
|
||||
and 0 < o.shape[0] <= 384
|
||||
and get_platform().is_blackwell
|
||||
)
|
||||
or (
|
||||
is_prefill
|
||||
and 4096 <= o.shape[0] <= 65536
|
||||
and get_platform().is_blackwell
|
||||
)
|
||||
)
|
||||
)
|
||||
and o.shape[1:] == (2, 4096)
|
||||
or hip_decode_verify
|
||||
or (hip_fast_path and is_prefill and 4096 <= o.shape[0] <= 65536)
|
||||
) and (
|
||||
o.shape[1:] == (2, 4096)
|
||||
and wo_a.shape == (2, 1024, 4096)
|
||||
and o.dtype == wo_a.dtype == torch.bfloat16
|
||||
and o.stride(2) == 1
|
||||
@@ -507,6 +526,11 @@ def _apply_wo_a_bf16_matmul(
|
||||
o.transpose(0, 1), wo_a.transpose(1, 2), out=result.transpose(0, 1)
|
||||
)
|
||||
return result
|
||||
# the 16-row decode kernels also serve V4.1's target-verify rows
|
||||
if (is_decode or is_target_verify) and hip_fast_path:
|
||||
y = _hip.wo_a_fp8_grid_matmul(o, wo_a, fp8_grid)
|
||||
if y is not None:
|
||||
return y
|
||||
if (
|
||||
is_decode
|
||||
and _wo_a_aiter_batched_gemm_enabled
|
||||
@@ -722,7 +746,11 @@ def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
|
||||
get_attn_backend().forward_low_ratio_sources(
|
||||
layer=layer,
|
||||
x=x[:real_num_tokens],
|
||||
q_lora=q_lora[:real_num_tokens],
|
||||
q_lora=(
|
||||
_hip.live_rows(q_lora, real_num_tokens)
|
||||
if _is_hip
|
||||
else q_lora[:real_num_tokens]
|
||||
),
|
||||
positions=positions[:real_num_tokens],
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
@@ -1102,7 +1130,11 @@ class MQALayer(MqaAttentionBase):
|
||||
self.register_buffer("sin_cache", sin_cache, persistent=False)
|
||||
|
||||
if alt_streams is not None and (
|
||||
(_is_cuda and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get())
|
||||
(
|
||||
# the gfx950 overlap is validated on V4.1 only; DSv4 on ROCm keeps one stream
|
||||
(_is_cuda or (_is_gfx95_supported and self.is_dsv41))
|
||||
and envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.get()
|
||||
)
|
||||
or (_is_npu and envs.SGLANG_NPU_USE_MULTI_STREAM.get())
|
||||
):
|
||||
self.alt_streams = alt_streams[:3]
|
||||
@@ -1173,9 +1205,7 @@ class MQALayer(MqaAttentionBase):
|
||||
prefix=add_prefix("attn_mqa", prefix),
|
||||
)
|
||||
|
||||
self.use_fused_qk_norm_rope = (
|
||||
_is_hip and envs.SGLANG_OPT_USE_FUSED_QK_NORM_ROPE.get()
|
||||
)
|
||||
self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self)
|
||||
# Static eligibility; token count and wo_b's output format are checked
|
||||
# in forward, after weights have loaded.
|
||||
self.use_fused_wo_a = (
|
||||
@@ -1188,6 +1218,8 @@ class MQALayer(MqaAttentionBase):
|
||||
and self.wo_a.weight.shape == (self.n_local_groups * self.o_lora_rank, 4096)
|
||||
and (self.n_local_groups, self.o_lora_rank) == (2, 1024)
|
||||
)
|
||||
if _is_hip:
|
||||
_hip.init_mqa_layer(self, quant_config)
|
||||
|
||||
# KV cache write is always fused into the K kernel
|
||||
# (`_compute_kv_to_cache`), so the legacy "overlap store cache" flag
|
||||
@@ -1256,6 +1288,8 @@ class MQALayer(MqaAttentionBase):
|
||||
self, q: torch.Tensor
|
||||
) -> Tuple[torch.Tensor, torch.Tensor | Mxfp8SwizzledInput]:
|
||||
# The indexer needs the BF16 normalized row; wq_b needs the quantized one.
|
||||
if _is_hip:
|
||||
return _hip.q_norm_for_wq_b(self, q)
|
||||
method = self.wq_b.quant_method
|
||||
if (
|
||||
_is_cuda
|
||||
@@ -1310,13 +1344,13 @@ class MQALayer(MqaAttentionBase):
|
||||
q = q.view(-1, self.n_local_heads, self.head_dim)
|
||||
if not self.q_head_norm:
|
||||
if (
|
||||
_is_cuda
|
||||
(_is_cuda or _is_gfx95_supported)
|
||||
and q_out is not None
|
||||
and (
|
||||
0 < q.shape[0] <= 8
|
||||
or (
|
||||
self.is_dsv41
|
||||
and get_platform().is_blackwell
|
||||
and (get_platform().is_blackwell or _is_gfx95_supported)
|
||||
and self.n_local_heads == 16
|
||||
and 4096 <= q.shape[0] <= 65536
|
||||
)
|
||||
@@ -1354,13 +1388,16 @@ class MQALayer(MqaAttentionBase):
|
||||
forward_batch: ForwardBatch,
|
||||
attn_backend,
|
||||
qkv_a: Optional[torch.Tensor] = None,
|
||||
q_rope: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""Fused: rmsnorm + RoPE + write directly to FlashMLA paged cache.
|
||||
|
||||
Replaces the bf16-kv-intermediate path. Used everywhere except the DSA
|
||||
prefill-CP case (which needs bf16 kv for the cross-rank all-gather).
|
||||
q_rope ([T, H, head_dim]) has its query heads roped by the same launch.
|
||||
"""
|
||||
if envs.SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE.get():
|
||||
assert q_rope is None
|
||||
# Quantize the nope payload from bf16-rounded values (the fused
|
||||
# kernel quantizes from fp32 registers; the bf16 rounding moves
|
||||
# values across fp8 bins relative to bf16-sourced consumers).
|
||||
@@ -1384,6 +1421,8 @@ class MQALayer(MqaAttentionBase):
|
||||
eps=self.eps,
|
||||
freqs_cis=self.freqs_cis,
|
||||
positions=positions,
|
||||
# the query RoPE fused into this store is HIP only
|
||||
**({} if q_rope is None else {"q": q_rope}),
|
||||
)
|
||||
|
||||
def _compute_kv_bf16(
|
||||
@@ -1724,8 +1763,8 @@ class MQALayer(MqaAttentionBase):
|
||||
dtype=x.dtype,
|
||||
)
|
||||
else:
|
||||
q_lora = self.q_norm(q_lora)
|
||||
q = self._compute_q_b(q_lora, positions, q_out)
|
||||
q_lora, q_for_wqb = self._normalize_q_lora(q_lora)
|
||||
q = self._compute_q_b(q_for_wqb, positions, q_out)
|
||||
self._compute_kv_to_cache(
|
||||
x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a
|
||||
)
|
||||
@@ -1982,7 +2021,13 @@ class MQALayer(MqaAttentionBase):
|
||||
q_out.copy_(q)
|
||||
else:
|
||||
q_lora, q_for_wqb = self._normalize_q_lora(q_lora)
|
||||
q = self._compute_q_b(q_for_wqb, positions, q_out)
|
||||
if _is_hip:
|
||||
q, q_lora, fuse_q_rope = _hip.compute_q_b(
|
||||
self, q_lora, q_for_wqb, positions, q_out, unified, use_cp
|
||||
)
|
||||
else:
|
||||
q = self._compute_q_b(q_for_wqb, positions, q_out)
|
||||
fuse_q_rope = False
|
||||
if unified:
|
||||
# unified_kv prefill: keep bf16 kv; the backend writes
|
||||
# the ring AFTER attention (2-source path).
|
||||
@@ -2028,7 +2073,12 @@ class MQALayer(MqaAttentionBase):
|
||||
kv = None
|
||||
else:
|
||||
self._compute_kv_to_cache(
|
||||
x_linear, positions, forward_batch, attn_backend, qkv_a=qkv_a
|
||||
x_linear,
|
||||
positions,
|
||||
forward_batch,
|
||||
attn_backend,
|
||||
qkv_a=qkv_a,
|
||||
q_rope=q if fuse_q_rope else None,
|
||||
)
|
||||
kv = None
|
||||
|
||||
@@ -2132,8 +2182,7 @@ class MQALayer(MqaAttentionBase):
|
||||
)
|
||||
|
||||
low_ratio_multi_stream = (
|
||||
_is_cuda
|
||||
and get_platform().is_blackwell
|
||||
((_is_cuda and get_platform().is_blackwell) or _is_gfx95_supported)
|
||||
and self.compress_ratio in (1, 2)
|
||||
and self.alt_streams is not None
|
||||
and (
|
||||
@@ -2141,8 +2190,11 @@ class MQALayer(MqaAttentionBase):
|
||||
or (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
# Other MXFP8 backends may share mutable GEMM workspace.
|
||||
and getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None)
|
||||
== Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL
|
||||
and (
|
||||
_is_gfx95_supported
|
||||
or getattr(self.wq_b.quant_method, "mxfp8_dense_backend", None)
|
||||
== Mxfp8DenseGemmBackend.FLASHINFER_CUTEDSL
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -2223,7 +2275,11 @@ class MQALayer(MqaAttentionBase):
|
||||
k_rope = rope_pool.new_empty((x.shape[0], rope_pool.shape[-1]))
|
||||
kernel_num_heads = self.n_local_heads
|
||||
else:
|
||||
kernel_num_heads = self._kernel_num_heads(x.shape[0])
|
||||
kernel_num_heads = (
|
||||
self.n_local_heads
|
||||
if _is_hip and not unified and _hip.skip_head_pad(self)
|
||||
else self._kernel_num_heads(x.shape[0])
|
||||
)
|
||||
if kernel_num_heads != self.n_local_heads:
|
||||
# Backends without an exact-head specialization retain the existing
|
||||
# padded shape. attn_sink is sliced to this rank and padded to match.
|
||||
@@ -2318,8 +2374,21 @@ class MQALayer(MqaAttentionBase):
|
||||
# its normal causally-indexed store from attn_k = kv.
|
||||
attn_k = kv if kv is not None else q
|
||||
|
||||
# HIP: the attention kernel may apply the inverse RoPE itself; the else-branch below keeps it
|
||||
inv_rope = (
|
||||
_hip.attention_inv_rope(
|
||||
self,
|
||||
positions,
|
||||
forward_batch,
|
||||
unified,
|
||||
wo_a_applies_inv_rope=self.wo_a_fp8
|
||||
and _wo_a_fp8_mxscale_fused_invrope is not None,
|
||||
)
|
||||
if _is_hip
|
||||
else None
|
||||
)
|
||||
if unified:
|
||||
# only the HIP radix backend takes these two; passing them always would
|
||||
# only the HIP radix backend takes these; passing them always would
|
||||
# leave non-ROCm depending on the **_ in its forward() to drop them, and
|
||||
# no test on that side would notice if the **_ went away
|
||||
rope_kwargs = {}
|
||||
@@ -2327,6 +2396,8 @@ class MQALayer(MqaAttentionBase):
|
||||
rope_kwargs["q_rope"] = q_rope
|
||||
if k_rope is not None:
|
||||
rope_kwargs["k_rope"] = k_rope
|
||||
if inv_rope is not None:
|
||||
rope_kwargs["inv_rope"] = inv_rope
|
||||
o = attn_backend.forward(
|
||||
q=q_out if q_out is not None else q,
|
||||
k=attn_k,
|
||||
@@ -2364,6 +2435,8 @@ class MQALayer(MqaAttentionBase):
|
||||
compress_ratio=self.compress_ratio,
|
||||
attn_sink=attn_sink,
|
||||
save_kv_cache=save_kv_cache,
|
||||
# HIP only, for the same reason as rope_kwargs above
|
||||
**({} if inv_rope is None else {"inv_rope": inv_rope}),
|
||||
)
|
||||
o = o[:, tp_slice, :]
|
||||
if (
|
||||
@@ -2405,7 +2478,7 @@ class MQALayer(MqaAttentionBase):
|
||||
sin4,
|
||||
qk_nope_dim=self.qk_nope_head_dim,
|
||||
)
|
||||
elif not fuse_rope_wo_a:
|
||||
elif inv_rope is None and not fuse_rope_wo_a:
|
||||
# The fused path folds this in; it must not run twice.
|
||||
fused_rope_inplace(
|
||||
o[..., -self.qk_rope_head_dim :],
|
||||
@@ -2510,6 +2583,7 @@ class MQALayer(MqaAttentionBase):
|
||||
is_prefill=forward_batch.forward_mode.is_extend_without_speculative(),
|
||||
fast_path=self.is_dsv41,
|
||||
fuse_mxfp8_quant=fuse_mxfp8_quant,
|
||||
fp8_grid=_is_hip and _hip.wo_a_emits_fp8_grid(self),
|
||||
)
|
||||
else:
|
||||
o = _apply_gguf_grouped_wo_a(
|
||||
@@ -2519,16 +2593,21 @@ class MQALayer(MqaAttentionBase):
|
||||
self.o_lora_rank,
|
||||
)
|
||||
|
||||
return self._project_wo_b(o)
|
||||
|
||||
def _project_wo_b(self, o):
|
||||
from sglang.srt.layers.moe.mhc_post_fusion import current_mhc_post_fusion
|
||||
|
||||
mhc = current_mhc_post_fusion()
|
||||
o, _ = self.wo_b(
|
||||
o if isinstance(o, Mxfp8SwizzledInput) else o.flatten(1),
|
||||
o.flatten(1) if isinstance(o, torch.Tensor) else o,
|
||||
skip_all_reduce=mhc is not None,
|
||||
)
|
||||
if mhc is not None and mhc.overlap_only:
|
||||
mhc.start_stats_before_all_reduce()
|
||||
o = attn_tp_all_reduce(o)
|
||||
elif mhc is not None and _is_hip:
|
||||
_hip.apply_attention_mhc(o, mhc)
|
||||
elif mhc is not None:
|
||||
from sglang.kernels.ops.communication.all_reduce_mhc import (
|
||||
all_reduce_mhc_norm,
|
||||
@@ -2679,6 +2758,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
self.hc_pre_from_prev_sublayer = config.hc_pre_from_prev_sublayer
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
self.use_fused_mhc_post_pre = False
|
||||
if _is_hip:
|
||||
_hip.init_decoder_layer(self, quant_config)
|
||||
self.engram = None
|
||||
if engram_layout is not None and layer_id in engram_layout.layer_ids:
|
||||
self.engram = Engram(
|
||||
@@ -2759,6 +2840,25 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
)
|
||||
self._hc_ffn_bf16_parts = split_bf16_hc_weight(self.hc_ffn_fn.data)
|
||||
|
||||
def _input_norm(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
allow_aiter_quant: bool = True,
|
||||
coefficients=None,
|
||||
) -> Tuple[torch.Tensor, Optional[Tuple]]:
|
||||
"""input_layernorm(hidden_states) as (the bf16 norm attention reads, the pre-quantized
|
||||
operand of its dense projections or None); coefficients is a ROCm boundary's pending
|
||||
reduce + sinkhorn."""
|
||||
if _is_hip:
|
||||
return _hip.input_norm(
|
||||
self,
|
||||
hidden_states,
|
||||
allow_aiter_quant,
|
||||
coefficients,
|
||||
_fused_rmsnorm_fp8_quant,
|
||||
)
|
||||
return self.input_layernorm(hidden_states), None
|
||||
|
||||
def hc_pre(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
@@ -2959,6 +3059,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
):
|
||||
return mhc_post_split_h(x, residual, post, comb)
|
||||
|
||||
if _is_hip:
|
||||
y = _hip.hc_post(self, x, residual, post, comb)
|
||||
if y is not None:
|
||||
return y
|
||||
|
||||
if envs.SGLANG_OPT_USE_FLASHINFER_MHC.get():
|
||||
from flashinfer.mhc import mhc_post
|
||||
|
||||
@@ -3053,15 +3158,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# Triton fused post+pre (gfx95 small-batch or gfx1250) returns
|
||||
# norm_fused=False — the input layernorm is NOT folded.
|
||||
# gfx95 takes the fp8-quant path; gfx1250 takes plain layernorm.
|
||||
if _use_aiter and _is_gfx95_supported:
|
||||
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
|
||||
hidden_states,
|
||||
self.input_layernorm.weight,
|
||||
self.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
x_quant = None
|
||||
hidden_states, x_quant = self._input_norm(
|
||||
hidden_states, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x_quant = None
|
||||
else:
|
||||
@@ -3078,15 +3177,10 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
if _use_aiter and _is_gfx95_supported:
|
||||
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
|
||||
hidden_states,
|
||||
self.input_layernorm.weight,
|
||||
self.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
x_quant = None
|
||||
# gfx1250 takes plain layernorm here
|
||||
hidden_states, x_quant = self._input_norm(
|
||||
hidden_states, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x_quant = None
|
||||
else:
|
||||
@@ -3100,15 +3194,10 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
if _use_aiter and _is_gfx95_supported:
|
||||
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
|
||||
hidden_states,
|
||||
self.input_layernorm.weight,
|
||||
self.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
x_quant = None
|
||||
# gfx1250 takes plain layernorm here
|
||||
hidden_states, x_quant = self._input_norm(
|
||||
hidden_states, allow_aiter_quant=_is_gfx95_supported
|
||||
)
|
||||
else:
|
||||
x_quant = None
|
||||
|
||||
@@ -3290,10 +3379,16 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
|
||||
if (
|
||||
x.is_cuda
|
||||
and torch.version.cuda is not None
|
||||
and (
|
||||
get_platform().is_blackwell
|
||||
or (get_platform().is_sm90 and x.shape[0] == 1)
|
||||
# gfx950: the fused Triton port wins at every row count (MI350X)
|
||||
_is_gfx95_supported
|
||||
or (
|
||||
torch.version.cuda is not None
|
||||
and (
|
||||
get_platform().is_blackwell
|
||||
or (get_platform().is_sm90 and x.shape[0] == 1)
|
||||
)
|
||||
)
|
||||
)
|
||||
and x.dtype == torch.bfloat16
|
||||
):
|
||||
@@ -3944,15 +4039,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if not norm_fused:
|
||||
if _use_aiter and (_is_gfx95_supported or _is_gfx1250_supported):
|
||||
x_quant, hidden_states = _fused_rmsnorm_fp8_quant(
|
||||
hidden_states,
|
||||
self.input_layernorm.weight,
|
||||
self.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
x_quant = None
|
||||
hidden_states, x_quant = self._input_norm(hidden_states)
|
||||
else:
|
||||
x_quant = None
|
||||
|
||||
@@ -4433,14 +4520,20 @@ class DeepseekV4Model(nn.Module):
|
||||
precomputed_attn = None
|
||||
combined_attn = None
|
||||
normalized_attn = None
|
||||
# HIP: the fused boundary hands the next layer its FFN hc_post unapplied
|
||||
pending_post = None
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
if tail is not None and i == self.late_layer_start:
|
||||
combined_attn = None
|
||||
normalized_attn = None
|
||||
# Decode reaches back at most SWA_WINDOW positions.
|
||||
saved_full = attn_backend.enter_late_layer_tail(forward_batch)
|
||||
hidden_states, prev_pre, input_ids, input_ids_global = (
|
||||
tail.rows(hidden_states),
|
||||
if pending_post is not None:
|
||||
# the fused boundary carries the residual stream here, not in hidden_states
|
||||
pending_post = tuple(tail.rows(t) for t in pending_post)
|
||||
else:
|
||||
hidden_states = tail.rows(hidden_states)
|
||||
prev_pre, input_ids, input_ids_global = (
|
||||
tail.rows(prev_pre),
|
||||
tail.rows(input_ids),
|
||||
tail.rows(input_ids_global),
|
||||
@@ -4484,10 +4577,13 @@ class DeepseekV4Model(nn.Module):
|
||||
next_input = []
|
||||
# The next layer can consume a collapsed input only if no Engram
|
||||
# or row selection changes the residual between the two layers.
|
||||
# The HIP fused boundary carries the residual in pending_post and
|
||||
# leaves hidden_states None; it never consumes a collapsed input.
|
||||
next_combined = (
|
||||
[]
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
hidden_states is not None
|
||||
and self.config.model_type == "deepseek_v41"
|
||||
and (
|
||||
128 <= hidden_states.shape[0] <= 384
|
||||
or (
|
||||
@@ -4505,10 +4601,10 @@ class DeepseekV4Model(nn.Module):
|
||||
next_norm = self.layers[i + 1].input_layernorm
|
||||
if (
|
||||
self.config.model_type == "deepseek_v41"
|
||||
and get_platform().is_blackwell
|
||||
and i + 1 < self.end_layer
|
||||
and tail is None
|
||||
and hidden_states.is_cuda
|
||||
and get_platform().is_blackwell
|
||||
and 0 < hidden_states.shape[0] <= 8
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
@@ -4533,6 +4629,24 @@ class DeepseekV4Model(nn.Module):
|
||||
):
|
||||
next_norm = norm
|
||||
with ctx:
|
||||
if _is_hip and self.layers[i].hc_boundary_fused:
|
||||
hidden_states, prev_pre, pending_post = (
|
||||
_hip.forward_layer_fused_boundary(
|
||||
self,
|
||||
i,
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
input_ids=input_ids,
|
||||
forward_batch=forward_batch,
|
||||
input_ids_global=input_ids_global,
|
||||
prev_pre=prev_pre,
|
||||
pending_post=pending_post,
|
||||
capture_dspark=capture_dspark,
|
||||
)
|
||||
)
|
||||
# the fused boundary never publishes a collapsed input for the next layer
|
||||
precomputed_attn = combined_attn = normalized_attn = None
|
||||
continue
|
||||
hidden_states, prev_pre = self.layers[i].forward_hc_pre_from_prev(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -335,10 +335,12 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
and not get_parallel().enable_dp_attention
|
||||
and self.model_runner.pp_size == 1
|
||||
)
|
||||
# ROCm (V4.1 target only): inside a HIP graph the accept-site TP broadcasts
|
||||
# need the group's pynccl communicator
|
||||
if (
|
||||
(self._verify_planner.is_compact_mode or static_epilogue_supported)
|
||||
and self._decode_graph_allowed
|
||||
and is_cuda()
|
||||
and (is_cuda() or (is_cuda_alike() and target_is_dsv41))
|
||||
):
|
||||
self._verify_epilogue = DsparkVerifyEpilogue(
|
||||
max_bs=max(get_exec().graph.cuda_graph_config.decode.bs),
|
||||
|
||||
Reference in New Issue
Block a user