[AMD][Quark] Serve the Kimi-K3 MXFP4 checkpoint on ROCm (#40811)

Co-authored-by: kk <43161300+kkHuang-amd@users.noreply.github.com>
This commit is contained in:
Yu-Yun Chang
2026-09-30 18:14:15 -07:00
committed by GitHub
co-authored by kk
parent a702970ecb
commit 6af651ea0c
10 changed files with 402 additions and 11 deletions
@@ -15,7 +15,7 @@ _WARMED: set[tuple[int, float, float]] = set()
def enabled() -> bool:
return os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower() == "aiter"
return os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower() == "aiter"
def _ops():
@@ -17,6 +17,9 @@ from sglang.srt.layers.quantization.dequantization import (
dequantize_nvfp4,
)
from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
from sglang.srt.layers.quantization.mxfp4 import (
_aiter_situ_uses_gu_interleaved_weights,
)
from sglang.srt.layers.quantization.online_quantization import CopyNumelCounter
from sglang.srt.layers.quantization.quark.schemes import QuarkMoEScheme
from sglang.srt.layers.quantization.quark.utils import Nvfp4SourceConfig
@@ -44,8 +47,15 @@ _is_hip = is_hip()
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
_is_gfx95 = is_gfx95_supported()
_is_shuffle_moe_mxfp4 = _use_aiter and _is_gfx95
_aiter_k3_opt = _use_aiter and get_bool_env_var("SGLANG_AITER_K3_OPT")
if _use_aiter:
from aiter.ops.shuffle import moe_shuffle_scale, moe_shuffle_weight, shuffle_weight
from aiter.ops.shuffle import (
moe_shuffle_scale,
moe_shuffle_weight,
shuffle_scale_a16w4,
shuffle_weight,
shuffle_weight_a16w4,
)
from aiter.utility.fp4_utils import e8m0_shuffle
@@ -196,9 +206,15 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
)
return
# Generic AITER kernels align the packed W2 dimension to 128. For K3
# TP8 that changes the native 384 intermediate width (192 packed) into
# 512 (256 packed), adding 33% zero work and preventing the native-shape
# MXMoE tuning table from binding. The K3 FlyDSL path supports 128-wide
# alignment in the unpacked dimension, so retain the checkpoint shape.
pad_for_aiter = _use_aiter and not _aiter_k3_opt
w13_up_dim, w2_down_dim, weight_padded = get_moe_weight_sizes(
intermediate_size_per_partition,
is_aiter_moe=_use_aiter,
is_aiter_moe=pad_for_aiter,
is_concat=True,
is_packed=True,
)
@@ -820,6 +836,36 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
layer.w2_weight = torch.nn.Parameter(qw2_weight, requires_grad=False)
def _shuffle_gu_interleaved(self, layer: torch.nn.Module) -> None:
"""Preshuffle into the GU-interleaved layout the a16w4/a8w4 kernels read.
Scales stay 2-D; opus_moe rejects the 3-D form the a4w4 path keeps.
"""
num_experts = layer.w13_weight_scale.shape[0]
layer.w13_weight.data = shuffle_weight_a16w4(layer.w13_weight, 16, True)
w13_weight_scale = shuffle_scale_a16w4(
layer.w13_weight_scale.view(-1, layer.w13_weight_scale.shape[-1]),
num_experts,
True,
)
layer.w2_weight.data = shuffle_weight_a16w4(layer.w2_weight, 16, False)
w2_weight_scale = shuffle_scale_a16w4(
layer.w2_weight_scale.view(-1, layer.w2_weight_scale.shape[-1]),
num_experts,
False,
)
# Tagged so apply_weights can carry the flag across .view(float4_e2m1fn_x2)
# and aiter's fused_moe selects the preshuffle_on kernel family.
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
layer.w13_weight_scale = torch.nn.Parameter(
w13_weight_scale, requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, requires_grad=False)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if not getattr(self, "_owns_moe_runner", False):
raise RuntimeError(
@@ -839,6 +885,22 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
assert layer.w13_weight_scale.dtype == torch.uint8
assert layer.w2_weight_scale.dtype == torch.uint8
# AITER picks the SiTUv2 activation dtype at runtime: A8W4 and the
# A16W4 default read the preshuffled GU-interleaved layout, only A4W4
# reads the separated layout below. The mxfp4 MoE method agrees.
if (
_is_shuffle_moe_mxfp4
and not _is_gfx1250
and layer.moe_runner_config.activation == "situ"
and _aiter_situ_uses_gu_interleaved_weights()
):
self._shuffle_gu_interleaved(layer)
if hasattr(layer, "dispatcher"):
layer.dispatcher.set_quant_config(
{"weight_dtype": torch.float4_e2m1fn_x2}
)
return
# Pre-shuffle weight scales
if _is_gfx1250:
# gfx1250 grouped MoE GEMM consumes B-scales in the n32k4 layout.
@@ -94,6 +94,21 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
weight_scale = layer.weight_scale.data
if _is_hip and weight.shape[0] % 16 != 0:
# hipBLASLt needs the GEMM's N to be a multiple of 16 and
# torch._scaled_mm raises instead of padding. A narrow output
# partition cannot satisfy that at any batch size (Kimi-K3's KDA
# b_proj emits one beta channel per head, 12 at TP=8), so serve
# this linear dequantized; at that width it is memory-bound.
layer.weight = Parameter(
(weight.to(torch.float32) * weight_scale.view(-1, 1))
.to(self.out_dtype)
.t(),
requires_grad=False,
)
layer.weight_scale = None
layer.input_scale = None
return
if self.per_token:
weight_scale = weight_scale.view(-1, 1)
if use_aiter_bpreshuffle_gemm(weight.shape[0]):
@@ -182,6 +197,11 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
"RMSNorm+quant producer was not gated off by _is_block_scale_fp8 "
"(per-channel FP8 must fall through to the plain bf16 path)."
)
if _is_hip and layer.weight_scale is None:
# Dequantized at load; weight is [in, out] like the fp8 path's.
output = torch.matmul(x, layer.weight)
return output if bias is None else output + bias
return apply_fp8_linear(
x,
layer.weight,
+53 -2
View File
@@ -225,6 +225,15 @@ def _k3_bf16_gemm(
# chain + o_norm here.
def _is_unquantized_mergeable(weights: list[torch.Tensor]) -> bool:
"""Return whether these weights may be concatenated into one fused buffer.
_merge_weights_as_views cats .weight alone, so anything carrying a separate
scale tensor (per-channel FP8, packed MXFP4) must stay unfused."""
dtypes = {weight.dtype for weight in weights}
return len(dtypes) == 1 and dtypes.pop() in (torch.bfloat16, torch.float16)
def _merge_weights_as_views(
mods: list, pad_rows_to: int = 1
) -> tuple[torch.Tensor, list[int]]:
@@ -1850,6 +1859,11 @@ class KimiK3DeltaAttention(nn.Module):
else:
if any(getattr(mod, "weight", None) is None for mod in mods):
return
# ROCm Quark checkpoints: leave per-channel FP8 / MXFP4 weights on
# the unfused b_proj/f_a_proj GEMVs; the merged buffer would drop
# their scales.
if _is_hip and not _is_unquantized_mergeable([mod.weight for mod in mods]):
return
self._bfa_w, sizes = _merge_weights_as_views(mods, pad_rows_to=8)
self._bfa_f_b_w = self.f_b_proj.weight
self._bfa_fa_size, self._bfa_b_size = sizes
@@ -1905,6 +1919,11 @@ class KimiK3DeltaAttention(nn.Module):
ws = [m.weight for m in (self.fused_qkvg_proj, self.f_a_proj, self.b_proj)]
if not all(type(w.data) is torch.Tensor and w.dim() == 2 for w in ws):
return False
# Whitelist the dtype rather than only require the three to agree: the
# merged buffer carries only .weight, so quantized weights that happen to
# match each other still lose their per-channel scales.
if not _is_unquantized_mergeable(ws):
return False
return len({(w.dtype, w.shape[1]) for w in ws}) == 1
def _prepare_fused_decode(self) -> None:
@@ -1921,7 +1940,7 @@ class KimiK3DeltaAttention(nn.Module):
layer = self.attn
w = layer.conv_weights
f_b_weight = self.f_b_proj.weight
backend = os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower()
backend = os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower()
backend_available = (
backend == "aiter"
and kda_fused_decode_aiter_hip.available(f_b_weight.device)
@@ -3454,12 +3473,44 @@ class KimiK3LinearForCausalLM(nn.Module):
self_attn.use_deep_gemm_bmm = False
continue
kv_b_weight = _get_k3_dense_weight(self_attn.kv_b_proj)
scale_folded_into_weight = False
if _is_hip and kv_b_weight.dtype in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
):
scale = getattr(self_attn.kv_b_proj, "weight_scale", None)
if isinstance(scale, torch.Tensor) and scale.numel() > 1:
from sglang.srt.models.kimi_k3_rocm_quant import (
_k3_channel_fp8_to_bf16,
)
# Fold the per-channel scale while dim 0 is still the
# channel axis it indexes, i.e. before the head split.
kv_b_weight = _k3_channel_fp8_to_bf16(
self_attn.kv_b_proj, kv_b_weight
)
scale_folded_into_weight = True
w_kc, w_vc = kv_b_weight.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
if hasattr(self_attn.kv_b_proj, "weight_scale"):
kv_b_scale = getattr(self_attn.kv_b_proj, "weight_scale", None)
if _is_hip and (
scale_folded_into_weight
or not (
isinstance(kv_b_scale, torch.Tensor) and kv_b_scale.numel() == 1
)
):
# aiter's absorb GEMM dereferences w_scale as one scalar. A scale
# folded into the now-bf16 w_kc/w_vc, a vector the branch above
# could not fold, or the None quark leaves on a dequantized
# narrow partition must keep DeepseekV2AttentionMLA's 1.0
# default. Skip the assignment rather than reset afterwards:
# assigning a Parameter registers it, and nn.Module then refuses
# a float in its place.
pass
elif hasattr(self_attn.kv_b_proj, "weight_scale"):
self_attn.w_scale = self_attn.kv_b_proj.weight_scale
# Post-load: precompute the attn-res combined score weights BEFORE
@@ -0,0 +1,45 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Kimi-K3 ROCm weight preparation for Quark checkpoints.
Quark serializes K3 as per-output-channel FP8, which is not the layout the
AITER kernels want. The conversions live here; ``kimi_k3.py`` calls every entry
point behind ``_is_hip`` and keeps the checkpoint's own layout otherwise.
"""
import torch
from torch import nn
def _k3_channel_fp8_to_bf16(module: nn.Module, weight: torch.Tensor) -> torch.Tensor:
"""Dequantize a per-output-channel FP8 weight to dense bf16.
The aiter batched absorb GEMM takes ``w_scale`` as a single scalar (the
Triton kernel documents it as "per-batch scale for WQ with shape (1,)"), so
a per-channel vector cannot reach it. Requantizing to one per-tensor scale
would fit, but the scale axis is the GEMM's *contraction* axis, and on real
K3 weights that second quantization costs ~2.3% relative error against the
checkpoint (worst channel ~3.8%). Dequantizing instead costs ~0.12% and
lets the absorb run its bf16 path with w_scale left at the 1.0 default.
Must run while dim 0 is still the channel axis weight_scale indexes, i.e.
before the kv_b_proj head split.
"""
weight_scale = module.weight_scale
# Per-channel scale is 1D [out]; reshape so it broadcasts against [out, in].
if weight_scale.dim() == 1:
weight_scale = weight_scale.view(-1, 1)
return (weight.to(torch.float32) * weight_scale.to(torch.float32)).to(
torch.bfloat16
)