[AMD][Quark] Serve the Kimi-K3 MXFP4 checkpoint on ROCm (#40811)
Co-authored-by: kk <43161300+kkHuang-amd@users.noreply.github.com>
This commit is contained in:
@@ -15,7 +15,7 @@ _WARMED: set[tuple[int, float, float]] = set()
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower() == "aiter"
|
||||
return os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower() == "aiter"
|
||||
|
||||
|
||||
def _ops():
|
||||
|
||||
@@ -17,6 +17,9 @@ from sglang.srt.layers.quantization.dequantization import (
|
||||
dequantize_nvfp4,
|
||||
)
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
|
||||
from sglang.srt.layers.quantization.mxfp4 import (
|
||||
_aiter_situ_uses_gu_interleaved_weights,
|
||||
)
|
||||
from sglang.srt.layers.quantization.online_quantization import CopyNumelCounter
|
||||
from sglang.srt.layers.quantization.quark.schemes import QuarkMoEScheme
|
||||
from sglang.srt.layers.quantization.quark.utils import Nvfp4SourceConfig
|
||||
@@ -44,8 +47,15 @@ _is_hip = is_hip()
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
|
||||
_is_gfx95 = is_gfx95_supported()
|
||||
_is_shuffle_moe_mxfp4 = _use_aiter and _is_gfx95
|
||||
_aiter_k3_opt = _use_aiter and get_bool_env_var("SGLANG_AITER_K3_OPT")
|
||||
if _use_aiter:
|
||||
from aiter.ops.shuffle import moe_shuffle_scale, moe_shuffle_weight, shuffle_weight
|
||||
from aiter.ops.shuffle import (
|
||||
moe_shuffle_scale,
|
||||
moe_shuffle_weight,
|
||||
shuffle_scale_a16w4,
|
||||
shuffle_weight,
|
||||
shuffle_weight_a16w4,
|
||||
)
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
|
||||
@@ -196,9 +206,15 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
|
||||
)
|
||||
return
|
||||
|
||||
# Generic AITER kernels align the packed W2 dimension to 128. For K3
|
||||
# TP8 that changes the native 384 intermediate width (192 packed) into
|
||||
# 512 (256 packed), adding 33% zero work and preventing the native-shape
|
||||
# MXMoE tuning table from binding. The K3 FlyDSL path supports 128-wide
|
||||
# alignment in the unpacked dimension, so retain the checkpoint shape.
|
||||
pad_for_aiter = _use_aiter and not _aiter_k3_opt
|
||||
w13_up_dim, w2_down_dim, weight_padded = get_moe_weight_sizes(
|
||||
intermediate_size_per_partition,
|
||||
is_aiter_moe=_use_aiter,
|
||||
is_aiter_moe=pad_for_aiter,
|
||||
is_concat=True,
|
||||
is_packed=True,
|
||||
)
|
||||
@@ -820,6 +836,36 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
|
||||
|
||||
layer.w2_weight = torch.nn.Parameter(qw2_weight, requires_grad=False)
|
||||
|
||||
def _shuffle_gu_interleaved(self, layer: torch.nn.Module) -> None:
|
||||
"""Preshuffle into the GU-interleaved layout the a16w4/a8w4 kernels read.
|
||||
|
||||
Scales stay 2-D; opus_moe rejects the 3-D form the a4w4 path keeps.
|
||||
"""
|
||||
num_experts = layer.w13_weight_scale.shape[0]
|
||||
|
||||
layer.w13_weight.data = shuffle_weight_a16w4(layer.w13_weight, 16, True)
|
||||
w13_weight_scale = shuffle_scale_a16w4(
|
||||
layer.w13_weight_scale.view(-1, layer.w13_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
True,
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight_a16w4(layer.w2_weight, 16, False)
|
||||
w2_weight_scale = shuffle_scale_a16w4(
|
||||
layer.w2_weight_scale.view(-1, layer.w2_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
False,
|
||||
)
|
||||
|
||||
# Tagged so apply_weights can carry the flag across .view(float4_e2m1fn_x2)
|
||||
# and aiter's fused_moe selects the preshuffle_on kernel family.
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
w13_weight_scale, requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, requires_grad=False)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if not getattr(self, "_owns_moe_runner", False):
|
||||
raise RuntimeError(
|
||||
@@ -839,6 +885,22 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
|
||||
assert layer.w13_weight_scale.dtype == torch.uint8
|
||||
assert layer.w2_weight_scale.dtype == torch.uint8
|
||||
|
||||
# AITER picks the SiTUv2 activation dtype at runtime: A8W4 and the
|
||||
# A16W4 default read the preshuffled GU-interleaved layout, only A4W4
|
||||
# reads the separated layout below. The mxfp4 MoE method agrees.
|
||||
if (
|
||||
_is_shuffle_moe_mxfp4
|
||||
and not _is_gfx1250
|
||||
and layer.moe_runner_config.activation == "situ"
|
||||
and _aiter_situ_uses_gu_interleaved_weights()
|
||||
):
|
||||
self._shuffle_gu_interleaved(layer)
|
||||
if hasattr(layer, "dispatcher"):
|
||||
layer.dispatcher.set_quant_config(
|
||||
{"weight_dtype": torch.float4_e2m1fn_x2}
|
||||
)
|
||||
return
|
||||
|
||||
# Pre-shuffle weight scales
|
||||
if _is_gfx1250:
|
||||
# gfx1250 grouped MoE GEMM consumes B-scales in the n32k4 layout.
|
||||
|
||||
@@ -94,6 +94,21 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
|
||||
layer.input_scale = Parameter(input_scale, requires_grad=False)
|
||||
else:
|
||||
weight_scale = layer.weight_scale.data
|
||||
if _is_hip and weight.shape[0] % 16 != 0:
|
||||
# hipBLASLt needs the GEMM's N to be a multiple of 16 and
|
||||
# torch._scaled_mm raises instead of padding. A narrow output
|
||||
# partition cannot satisfy that at any batch size (Kimi-K3's KDA
|
||||
# b_proj emits one beta channel per head, 12 at TP=8), so serve
|
||||
# this linear dequantized; at that width it is memory-bound.
|
||||
layer.weight = Parameter(
|
||||
(weight.to(torch.float32) * weight_scale.view(-1, 1))
|
||||
.to(self.out_dtype)
|
||||
.t(),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = None
|
||||
layer.input_scale = None
|
||||
return
|
||||
if self.per_token:
|
||||
weight_scale = weight_scale.view(-1, 1)
|
||||
if use_aiter_bpreshuffle_gemm(weight.shape[0]):
|
||||
@@ -182,6 +197,11 @@ class QuarkW8A8Fp8(QuarkLinearScheme):
|
||||
"RMSNorm+quant producer was not gated off by _is_block_scale_fp8 "
|
||||
"(per-channel FP8 must fall through to the plain bf16 path)."
|
||||
)
|
||||
if _is_hip and layer.weight_scale is None:
|
||||
# Dequantized at load; weight is [in, out] like the fp8 path's.
|
||||
output = torch.matmul(x, layer.weight)
|
||||
return output if bias is None else output + bias
|
||||
|
||||
return apply_fp8_linear(
|
||||
x,
|
||||
layer.weight,
|
||||
|
||||
@@ -225,6 +225,15 @@ def _k3_bf16_gemm(
|
||||
# chain + o_norm here.
|
||||
|
||||
|
||||
def _is_unquantized_mergeable(weights: list[torch.Tensor]) -> bool:
|
||||
"""Return whether these weights may be concatenated into one fused buffer.
|
||||
|
||||
_merge_weights_as_views cats .weight alone, so anything carrying a separate
|
||||
scale tensor (per-channel FP8, packed MXFP4) must stay unfused."""
|
||||
dtypes = {weight.dtype for weight in weights}
|
||||
return len(dtypes) == 1 and dtypes.pop() in (torch.bfloat16, torch.float16)
|
||||
|
||||
|
||||
def _merge_weights_as_views(
|
||||
mods: list, pad_rows_to: int = 1
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
@@ -1850,6 +1859,11 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
else:
|
||||
if any(getattr(mod, "weight", None) is None for mod in mods):
|
||||
return
|
||||
# ROCm Quark checkpoints: leave per-channel FP8 / MXFP4 weights on
|
||||
# the unfused b_proj/f_a_proj GEMVs; the merged buffer would drop
|
||||
# their scales.
|
||||
if _is_hip and not _is_unquantized_mergeable([mod.weight for mod in mods]):
|
||||
return
|
||||
self._bfa_w, sizes = _merge_weights_as_views(mods, pad_rows_to=8)
|
||||
self._bfa_f_b_w = self.f_b_proj.weight
|
||||
self._bfa_fa_size, self._bfa_b_size = sizes
|
||||
@@ -1905,6 +1919,11 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
ws = [m.weight for m in (self.fused_qkvg_proj, self.f_a_proj, self.b_proj)]
|
||||
if not all(type(w.data) is torch.Tensor and w.dim() == 2 for w in ws):
|
||||
return False
|
||||
# Whitelist the dtype rather than only require the three to agree: the
|
||||
# merged buffer carries only .weight, so quantized weights that happen to
|
||||
# match each other still lose their per-channel scales.
|
||||
if not _is_unquantized_mergeable(ws):
|
||||
return False
|
||||
return len({(w.dtype, w.shape[1]) for w in ws}) == 1
|
||||
|
||||
def _prepare_fused_decode(self) -> None:
|
||||
@@ -1921,7 +1940,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
layer = self.attn
|
||||
w = layer.conv_weights
|
||||
f_b_weight = self.f_b_proj.weight
|
||||
backend = os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower()
|
||||
backend = os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower()
|
||||
backend_available = (
|
||||
backend == "aiter"
|
||||
and kda_fused_decode_aiter_hip.available(f_b_weight.device)
|
||||
@@ -3454,12 +3473,44 @@ class KimiK3LinearForCausalLM(nn.Module):
|
||||
self_attn.use_deep_gemm_bmm = False
|
||||
continue
|
||||
kv_b_weight = _get_k3_dense_weight(self_attn.kv_b_proj)
|
||||
scale_folded_into_weight = False
|
||||
if _is_hip and kv_b_weight.dtype in (
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
):
|
||||
scale = getattr(self_attn.kv_b_proj, "weight_scale", None)
|
||||
if isinstance(scale, torch.Tensor) and scale.numel() > 1:
|
||||
from sglang.srt.models.kimi_k3_rocm_quant import (
|
||||
_k3_channel_fp8_to_bf16,
|
||||
)
|
||||
|
||||
# Fold the per-channel scale while dim 0 is still the
|
||||
# channel axis it indexes, i.e. before the head split.
|
||||
kv_b_weight = _k3_channel_fp8_to_bf16(
|
||||
self_attn.kv_b_proj, kv_b_weight
|
||||
)
|
||||
scale_folded_into_weight = True
|
||||
w_kc, w_vc = kv_b_weight.unflatten(
|
||||
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
|
||||
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
|
||||
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
|
||||
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
|
||||
if hasattr(self_attn.kv_b_proj, "weight_scale"):
|
||||
kv_b_scale = getattr(self_attn.kv_b_proj, "weight_scale", None)
|
||||
if _is_hip and (
|
||||
scale_folded_into_weight
|
||||
or not (
|
||||
isinstance(kv_b_scale, torch.Tensor) and kv_b_scale.numel() == 1
|
||||
)
|
||||
):
|
||||
# aiter's absorb GEMM dereferences w_scale as one scalar. A scale
|
||||
# folded into the now-bf16 w_kc/w_vc, a vector the branch above
|
||||
# could not fold, or the None quark leaves on a dequantized
|
||||
# narrow partition must keep DeepseekV2AttentionMLA's 1.0
|
||||
# default. Skip the assignment rather than reset afterwards:
|
||||
# assigning a Parameter registers it, and nn.Module then refuses
|
||||
# a float in its place.
|
||||
pass
|
||||
elif hasattr(self_attn.kv_b_proj, "weight_scale"):
|
||||
self_attn.w_scale = self_attn.kv_b_proj.weight_scale
|
||||
|
||||
# Post-load: precompute the attn-res combined score weights BEFORE
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Kimi-K3 ROCm weight preparation for Quark checkpoints.
|
||||
|
||||
Quark serializes K3 as per-output-channel FP8, which is not the layout the
|
||||
AITER kernels want. The conversions live here; ``kimi_k3.py`` calls every entry
|
||||
point behind ``_is_hip`` and keeps the checkpoint's own layout otherwise.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
def _k3_channel_fp8_to_bf16(module: nn.Module, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Dequantize a per-output-channel FP8 weight to dense bf16.
|
||||
|
||||
The aiter batched absorb GEMM takes ``w_scale`` as a single scalar (the
|
||||
Triton kernel documents it as "per-batch scale for WQ with shape (1,)"), so
|
||||
a per-channel vector cannot reach it. Requantizing to one per-tensor scale
|
||||
would fit, but the scale axis is the GEMM's *contraction* axis, and on real
|
||||
K3 weights that second quantization costs ~2.3% relative error against the
|
||||
checkpoint (worst channel ~3.8%). Dequantizing instead costs ~0.12% and
|
||||
lets the absorb run its bf16 path with w_scale left at the 1.0 default.
|
||||
|
||||
Must run while dim 0 is still the channel axis weight_scale indexes, i.e.
|
||||
before the kv_b_proj head split.
|
||||
"""
|
||||
weight_scale = module.weight_scale
|
||||
# Per-channel scale is 1D [out]; reshape so it broadcasts against [out, in].
|
||||
if weight_scale.dim() == 1:
|
||||
weight_scale = weight_scale.view(-1, 1)
|
||||
return (weight.to(torch.float32) * weight_scale.to(torch.float32)).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
Reference in New Issue
Block a user