From 6af651ea0cb4da0f98f65f31c0830a8f1fa38221 Mon Sep 17 00:00:00 2001 From: Yu-Yun Chang Date: Thu, 1 Oct 2026 09:14:15 +0800 Subject: [PATCH] [AMD][Quark] Serve the Kimi-K3 MXFP4 checkpoint on ROCm (#40811) Co-authored-by: kk <43161300+kkHuang-amd@users.noreply.github.com> --- .../autoregressive/Moonshotai/Kimi-K3.mdx | 4 +- .../snippets/configs/moonshotai/kimi-k3.jsx | 4 +- .../attention/kda_fused_decode_aiter_hip.py | 2 +- .../quark/schemes/quark_w4a4_mxfp4_moe.py | 66 +++++++++- .../quark/schemes/quark_w8a8_fp8.py | 20 +++ python/sglang/srt/models/kimi_k3.py | 55 ++++++++- .../sglang/srt/models/kimi_k3_rocm_quant.py | 45 +++++++ .../kda_flydsl/test_kimi_k3_kda_decode.py | 4 +- .../quantization/test_quark_w8a8_fp8.py | 115 ++++++++++++++++++ .../unit/models/test_kimi_k3_rocm_quant.py | 98 +++++++++++++++ 10 files changed, 402 insertions(+), 11 deletions(-) create mode 100644 python/sglang/srt/models/kimi_k3_rocm_quant.py create mode 100644 test/registered/unit/layers/quantization/test_quark_w8a8_fp8.py create mode 100644 test/registered/unit/models/test_kimi_k3_rocm_quant.py diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 00aa1a1723..60edb6c5f8 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -105,7 +105,7 @@ SGLang revision `e7f7447333`, which includes the [DCP KV-free fix (#38941)](https://github.com/sgl-project/sglang/pull/38941). No source overlay is required for DCP. -Keep `SGLANG_K3_KDA_FUSED_BACKEND` unset with this image. The separate fused-KDA +Keep `SGLANG_ROCM_K3_KDA_FUSED_BACKEND` unset with this image. The separate fused-KDA opt-in requires the [deferred-gate fix (#39066)](https://github.com/sgl-project/sglang/pull/39066), which is not included in this image. This updated recipe remains **Final Verification In Progress**; the recorded speed numbers use their original @@ -221,7 +221,7 @@ Outside the two verified B300 1×8 `Unified` cells, no cell has a serving round |---|---|---| | `AITER_SITUV2_A8W4=1` | unset | SiTU v2 MoE with A8W4 activation quantization, on AITER's GU-interleaved preshuffled weight layout. The performance default the cell ships. | | `AITER_SITUV2_A4W4=1` | unset | A4W4 instead, on the generic separated shuffle layout. Numerically correct but slower than A8W4 (530.8 vs 537.3 tok/s median output on 8×MI35x). Setting **both** gives A8W4 precedence — SGLang follows AITER and keeps the GU-interleaved layout. | -| `SGLANG_K3_KDA_FUSED_BACKEND=aiter` | unset | Opt in to the fused ROCm KDA decode boundary: the `f_b` projection is deferred into the gfx950 FlyDSL kernel so decode fuses `f_b` + convolution + recurrent state update + gated RMSNorm. Any other value (or unset) keeps the unfused KDA path. | +| `SGLANG_ROCM_K3_KDA_FUSED_BACKEND=aiter` | unset | Opt in to the fused ROCm KDA decode boundary: the `f_b` projection is deferred into the gfx950 FlyDSL kernel so decode fuses `f_b` + convolution + recurrent state update + gated RMSNorm. Any other value (or unset) keeps the unfused KDA path. | | `SGLANG_K3_FLYDSL_SOURCE` | `auto` | Which FlyDSL implementation backs the fused decode: `auto` prefers SGLang's vendored kernels and falls back to the AITER module, `sglang` / `aiter` pin one. Leave it alone unless you are bisecting the two. | The fused KDA backend is fail-closed at two levels: it arms during model init only when the flag is exactly `aiter` **and** the gfx950 FlyDSL kernels are importable, and each decode step re-validates shapes, dtypes, strides, state indices, and output buffers before dispatch — anything unexpected falls back to the stock KDA implementation rather than erroring. Batch size 2 automatically picks a separately validated kernel schedule; every other batch keeps the original build options. Measured on a 69-layer graph the fused boundary is 9.20 → 8.38 µs/layer (−8.9%), with GSM8K 1319 at 0.950. diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 0c821ee348..f1f2594976 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -984,10 +984,10 @@ export const config = { // dispatch. Env var, not a flag, so it emits via env/stripEnv. id: "kdaFusedDecode", title: "Fused KDA Decode (AMD gfx950)", showWhen: (b) => ["mi350x", "mi355x"].includes(b.hw), - stripEnv: ["SGLANG_K3_KDA_FUSED_BACKEND"], + stripEnv: ["SGLANG_ROCM_K3_KDA_FUSED_BACKEND"], options: [ { id: "off", label: "Off" }, - { id: "aiter", label: "On (AITER fused boundary)", env: ["SGLANG_K3_KDA_FUSED_BACKEND=aiter"] }, + { id: "aiter", label: "On (AITER fused boundary)", env: ["SGLANG_ROCM_K3_KDA_FUSED_BACKEND=aiter"] }, ], }, { diff --git a/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py b/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py index b192a2a73f..c82dde7fb5 100644 --- a/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py +++ b/python/sglang/kernels/ops/attention/kda_fused_decode_aiter_hip.py @@ -15,7 +15,7 @@ _WARMED: set[tuple[int, float, float]] = set() def enabled() -> bool: - return os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower() == "aiter" + return os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower() == "aiter" def _ops(): diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py index 5ce7cb57e9..b2dfbe7845 100644 --- a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py @@ -17,6 +17,9 @@ from sglang.srt.layers.quantization.dequantization import ( dequantize_nvfp4, ) from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod +from sglang.srt.layers.quantization.mxfp4 import ( + _aiter_situ_uses_gu_interleaved_weights, +) from sglang.srt.layers.quantization.online_quantization import CopyNumelCounter from sglang.srt.layers.quantization.quark.schemes import QuarkMoEScheme from sglang.srt.layers.quantization.quark.utils import Nvfp4SourceConfig @@ -44,8 +47,15 @@ _is_hip = is_hip() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip _is_gfx95 = is_gfx95_supported() _is_shuffle_moe_mxfp4 = _use_aiter and _is_gfx95 +_aiter_k3_opt = _use_aiter and get_bool_env_var("SGLANG_AITER_K3_OPT") if _use_aiter: - from aiter.ops.shuffle import moe_shuffle_scale, moe_shuffle_weight, shuffle_weight + from aiter.ops.shuffle import ( + moe_shuffle_scale, + moe_shuffle_weight, + shuffle_scale_a16w4, + shuffle_weight, + shuffle_weight_a16w4, + ) from aiter.utility.fp4_utils import e8m0_shuffle @@ -196,9 +206,15 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme): ) return + # Generic AITER kernels align the packed W2 dimension to 128. For K3 + # TP8 that changes the native 384 intermediate width (192 packed) into + # 512 (256 packed), adding 33% zero work and preventing the native-shape + # MXMoE tuning table from binding. The K3 FlyDSL path supports 128-wide + # alignment in the unpacked dimension, so retain the checkpoint shape. + pad_for_aiter = _use_aiter and not _aiter_k3_opt w13_up_dim, w2_down_dim, weight_padded = get_moe_weight_sizes( intermediate_size_per_partition, - is_aiter_moe=_use_aiter, + is_aiter_moe=pad_for_aiter, is_concat=True, is_packed=True, ) @@ -820,6 +836,36 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme): layer.w2_weight = torch.nn.Parameter(qw2_weight, requires_grad=False) + def _shuffle_gu_interleaved(self, layer: torch.nn.Module) -> None: + """Preshuffle into the GU-interleaved layout the a16w4/a8w4 kernels read. + + Scales stay 2-D; opus_moe rejects the 3-D form the a4w4 path keeps. + """ + num_experts = layer.w13_weight_scale.shape[0] + + layer.w13_weight.data = shuffle_weight_a16w4(layer.w13_weight, 16, True) + w13_weight_scale = shuffle_scale_a16w4( + layer.w13_weight_scale.view(-1, layer.w13_weight_scale.shape[-1]), + num_experts, + True, + ) + layer.w2_weight.data = shuffle_weight_a16w4(layer.w2_weight, 16, False) + w2_weight_scale = shuffle_scale_a16w4( + layer.w2_weight_scale.view(-1, layer.w2_weight_scale.shape[-1]), + num_experts, + False, + ) + + # Tagged so apply_weights can carry the flag across .view(float4_e2m1fn_x2) + # and aiter's fused_moe selects the preshuffle_on kernel family. + layer.w13_weight.is_shuffled = True + layer.w2_weight.is_shuffled = True + + layer.w13_weight_scale = torch.nn.Parameter( + w13_weight_scale, requires_grad=False + ) + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, requires_grad=False) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: if not getattr(self, "_owns_moe_runner", False): raise RuntimeError( @@ -839,6 +885,22 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme): assert layer.w13_weight_scale.dtype == torch.uint8 assert layer.w2_weight_scale.dtype == torch.uint8 + # AITER picks the SiTUv2 activation dtype at runtime: A8W4 and the + # A16W4 default read the preshuffled GU-interleaved layout, only A4W4 + # reads the separated layout below. The mxfp4 MoE method agrees. + if ( + _is_shuffle_moe_mxfp4 + and not _is_gfx1250 + and layer.moe_runner_config.activation == "situ" + and _aiter_situ_uses_gu_interleaved_weights() + ): + self._shuffle_gu_interleaved(layer) + if hasattr(layer, "dispatcher"): + layer.dispatcher.set_quant_config( + {"weight_dtype": torch.float4_e2m1fn_x2} + ) + return + # Pre-shuffle weight scales if _is_gfx1250: # gfx1250 grouped MoE GEMM consumes B-scales in the n32k4 layout. diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 06b501d10b..be775e7aac 100644 --- a/python/sglang/srt/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -94,6 +94,21 @@ class QuarkW8A8Fp8(QuarkLinearScheme): layer.input_scale = Parameter(input_scale, requires_grad=False) else: weight_scale = layer.weight_scale.data + if _is_hip and weight.shape[0] % 16 != 0: + # hipBLASLt needs the GEMM's N to be a multiple of 16 and + # torch._scaled_mm raises instead of padding. A narrow output + # partition cannot satisfy that at any batch size (Kimi-K3's KDA + # b_proj emits one beta channel per head, 12 at TP=8), so serve + # this linear dequantized; at that width it is memory-bound. + layer.weight = Parameter( + (weight.to(torch.float32) * weight_scale.view(-1, 1)) + .to(self.out_dtype) + .t(), + requires_grad=False, + ) + layer.weight_scale = None + layer.input_scale = None + return if self.per_token: weight_scale = weight_scale.view(-1, 1) if use_aiter_bpreshuffle_gemm(weight.shape[0]): @@ -182,6 +197,11 @@ class QuarkW8A8Fp8(QuarkLinearScheme): "RMSNorm+quant producer was not gated off by _is_block_scale_fp8 " "(per-channel FP8 must fall through to the plain bf16 path)." ) + if _is_hip and layer.weight_scale is None: + # Dequantized at load; weight is [in, out] like the fp8 path's. + output = torch.matmul(x, layer.weight) + return output if bias is None else output + bias + return apply_fp8_linear( x, layer.weight, diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index b77edd075b..a858db8176 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -225,6 +225,15 @@ def _k3_bf16_gemm( # chain + o_norm here. +def _is_unquantized_mergeable(weights: list[torch.Tensor]) -> bool: + """Return whether these weights may be concatenated into one fused buffer. + + _merge_weights_as_views cats .weight alone, so anything carrying a separate + scale tensor (per-channel FP8, packed MXFP4) must stay unfused.""" + dtypes = {weight.dtype for weight in weights} + return len(dtypes) == 1 and dtypes.pop() in (torch.bfloat16, torch.float16) + + def _merge_weights_as_views( mods: list, pad_rows_to: int = 1 ) -> tuple[torch.Tensor, list[int]]: @@ -1850,6 +1859,11 @@ class KimiK3DeltaAttention(nn.Module): else: if any(getattr(mod, "weight", None) is None for mod in mods): return + # ROCm Quark checkpoints: leave per-channel FP8 / MXFP4 weights on + # the unfused b_proj/f_a_proj GEMVs; the merged buffer would drop + # their scales. + if _is_hip and not _is_unquantized_mergeable([mod.weight for mod in mods]): + return self._bfa_w, sizes = _merge_weights_as_views(mods, pad_rows_to=8) self._bfa_f_b_w = self.f_b_proj.weight self._bfa_fa_size, self._bfa_b_size = sizes @@ -1905,6 +1919,11 @@ class KimiK3DeltaAttention(nn.Module): ws = [m.weight for m in (self.fused_qkvg_proj, self.f_a_proj, self.b_proj)] if not all(type(w.data) is torch.Tensor and w.dim() == 2 for w in ws): return False + # Whitelist the dtype rather than only require the three to agree: the + # merged buffer carries only .weight, so quantized weights that happen to + # match each other still lose their per-channel scales. + if not _is_unquantized_mergeable(ws): + return False return len({(w.dtype, w.shape[1]) for w in ws}) == 1 def _prepare_fused_decode(self) -> None: @@ -1921,7 +1940,7 @@ class KimiK3DeltaAttention(nn.Module): layer = self.attn w = layer.conv_weights f_b_weight = self.f_b_proj.weight - backend = os.environ.get("SGLANG_K3_KDA_FUSED_BACKEND", "").lower() + backend = os.environ.get("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "").lower() backend_available = ( backend == "aiter" and kda_fused_decode_aiter_hip.available(f_b_weight.device) @@ -3454,12 +3473,44 @@ class KimiK3LinearForCausalLM(nn.Module): self_attn.use_deep_gemm_bmm = False continue kv_b_weight = _get_k3_dense_weight(self_attn.kv_b_proj) + scale_folded_into_weight = False + if _is_hip and kv_b_weight.dtype in ( + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ): + scale = getattr(self_attn.kv_b_proj, "weight_scale", None) + if isinstance(scale, torch.Tensor) and scale.numel() > 1: + from sglang.srt.models.kimi_k3_rocm_quant import ( + _k3_channel_fp8_to_bf16, + ) + + # Fold the per-channel scale while dim 0 is still the + # channel axis it indexes, i.e. before the head split. + kv_b_weight = _k3_channel_fp8_to_bf16( + self_attn.kv_b_proj, kv_b_weight + ) + scale_folded_into_weight = True w_kc, w_vc = kv_b_weight.unflatten( 0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim) ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2) self_attn.w_vc = w_vc.contiguous().transpose(1, 2) - if hasattr(self_attn.kv_b_proj, "weight_scale"): + kv_b_scale = getattr(self_attn.kv_b_proj, "weight_scale", None) + if _is_hip and ( + scale_folded_into_weight + or not ( + isinstance(kv_b_scale, torch.Tensor) and kv_b_scale.numel() == 1 + ) + ): + # aiter's absorb GEMM dereferences w_scale as one scalar. A scale + # folded into the now-bf16 w_kc/w_vc, a vector the branch above + # could not fold, or the None quark leaves on a dequantized + # narrow partition must keep DeepseekV2AttentionMLA's 1.0 + # default. Skip the assignment rather than reset afterwards: + # assigning a Parameter registers it, and nn.Module then refuses + # a float in its place. + pass + elif hasattr(self_attn.kv_b_proj, "weight_scale"): self_attn.w_scale = self_attn.kv_b_proj.weight_scale # Post-load: precompute the attn-res combined score weights BEFORE diff --git a/python/sglang/srt/models/kimi_k3_rocm_quant.py b/python/sglang/srt/models/kimi_k3_rocm_quant.py new file mode 100644 index 0000000000..d1ef8c9e0e --- /dev/null +++ b/python/sglang/srt/models/kimi_k3_rocm_quant.py @@ -0,0 +1,45 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Kimi-K3 ROCm weight preparation for Quark checkpoints. + +Quark serializes K3 as per-output-channel FP8, which is not the layout the +AITER kernels want. The conversions live here; ``kimi_k3.py`` calls every entry +point behind ``_is_hip`` and keeps the checkpoint's own layout otherwise. +""" + +import torch +from torch import nn + + +def _k3_channel_fp8_to_bf16(module: nn.Module, weight: torch.Tensor) -> torch.Tensor: + """Dequantize a per-output-channel FP8 weight to dense bf16. + + The aiter batched absorb GEMM takes ``w_scale`` as a single scalar (the + Triton kernel documents it as "per-batch scale for WQ with shape (1,)"), so + a per-channel vector cannot reach it. Requantizing to one per-tensor scale + would fit, but the scale axis is the GEMM's *contraction* axis, and on real + K3 weights that second quantization costs ~2.3% relative error against the + checkpoint (worst channel ~3.8%). Dequantizing instead costs ~0.12% and + lets the absorb run its bf16 path with w_scale left at the 1.0 default. + + Must run while dim 0 is still the channel axis weight_scale indexes, i.e. + before the kv_b_proj head split. + """ + weight_scale = module.weight_scale + # Per-channel scale is 1D [out]; reshape so it broadcasts against [out, in]. + if weight_scale.dim() == 1: + weight_scale = weight_scale.view(-1, 1) + return (weight.to(torch.float32) * weight_scale.to(torch.float32)).to( + torch.bfloat16 + ) diff --git a/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py index 6ef7948d58..ca1706b8ba 100644 --- a/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py +++ b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py @@ -516,9 +516,9 @@ def test_decode_api_rejects_invalid_input_rank() -> None: def test_fused_kda_backend_is_opt_in(monkeypatch): from sglang.kernels.ops.attention import kda_fused_decode_aiter_hip - monkeypatch.delenv("SGLANG_K3_KDA_FUSED_BACKEND", raising=False) + monkeypatch.delenv("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", raising=False) assert not kda_fused_decode_aiter_hip.enabled() - monkeypatch.setenv("SGLANG_K3_KDA_FUSED_BACKEND", "aiter") + monkeypatch.setenv("SGLANG_ROCM_K3_KDA_FUSED_BACKEND", "aiter") assert kda_fused_decode_aiter_hip.enabled() diff --git a/test/registered/unit/layers/quantization/test_quark_w8a8_fp8.py b/test/registered/unit/layers/quantization/test_quark_w8a8_fp8.py new file mode 100644 index 0000000000..f8665babc4 --- /dev/null +++ b/test/registered/unit/layers/quantization/test_quark_w8a8_fp8.py @@ -0,0 +1,115 @@ +"""Unit tests for srt/layers/quantization/quark/schemes/quark_w8a8_fp8 on ROCm.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.quantization.quark.schemes.quark_w8a8_fp8 import QuarkW8A8Fp8 +from sglang.test.test_utils import CustomTestCase + +_MODULE = "sglang.srt.layers.quantization.quark.schemes.quark_w8a8_fp8" + + +def _per_channel_scheme() -> QuarkW8A8Fp8: + return QuarkW8A8Fp8( + weight_config={"qscheme": "per_channel"}, + input_config={"is_dynamic": True, "qscheme": "per_channel"}, + ) + + +def _quantized_layer(out_features: int, in_features: int = 32): + """A per-output-channel FP8 linear, the layout Quark serializes K3 with.""" + torch.manual_seed(0) + dense = torch.randn(out_features, in_features, dtype=torch.float32) + scale = dense.abs().amax(dim=1, keepdim=True) / 448.0 + weight = (dense / scale).to(torch.float8_e4m3fn) + layer = SimpleNamespace( + weight=torch.nn.Parameter(weight, requires_grad=False), + weight_scale=torch.nn.Parameter(scale.squeeze(1), requires_grad=False), + input_scale=None, + logical_widths=[out_features], + ) + return layer, weight, scale + + +class TestNarrowOutputPartitionFp8(CustomTestCase): + """hipBLASLt requires the GEMM's N to be a multiple of 16 and + ``torch._scaled_mm`` raises rather than padding, so a per-channel FP8 linear + whose output partition is narrower than that cannot be served quantized at + any batch size. K3's KDA ``b_proj`` emits one beta channel per head (12 at + TP=8) and hit exactly that. Such a layer is dequantized at load instead.""" + + def setUp(self): + self.scheme = _per_channel_scheme() + # is_hip()/is_fp8_fnuz() are evaluated at import time; pin both so the + # ROCm branch under test runs on any CI machine. + patcher = patch.multiple(_MODULE, _is_hip=True, _is_fp8_fnuz=False) + patcher.start() + self.addCleanup(patcher.stop) + + def test_narrow_partition_is_dequantized_at_load(self): + layer, weight, scale = _quantized_layer(out_features=12) + + self.scheme.process_weights_after_loading(layer) + + self.assertEqual(layer.weight.dtype, self.scheme.out_dtype) + # Dequantized weights fold the scale in, so keeping it would double-scale. + self.assertIsNone(layer.weight_scale) + self.assertIsNone(layer.input_scale) + # apply_fp8_linear consumes [in, out]; the dense fallback must match it. + self.assertEqual(tuple(layer.weight.shape), (32, 12)) + expected = (weight.to(torch.float32) * scale).to(self.scheme.out_dtype).t() + torch.testing.assert_close(layer.weight, expected) + + def test_aligned_partition_stays_quantized(self): + """The negative branch: widths hipBLASLt can serve keep their FP8 + weights, so the guard cannot degrade into dequantizing everything.""" + layer, _, _ = _quantized_layer(out_features=16) + + with patch(f"{_MODULE}.use_aiter_bpreshuffle_gemm", return_value=False): + self.scheme.process_weights_after_loading(layer) + + self.assertEqual(layer.weight.dtype, torch.float8_e4m3fn) + self.assertIsNotNone(layer.weight_scale) + self.assertEqual(layer.weight_scale.numel(), 16) + + def test_dequantized_layer_is_served_by_a_dense_matmul(self): + layer, weight, scale = _quantized_layer(out_features=12) + self.scheme.process_weights_after_loading(layer) + x = torch.randn(4, 32, dtype=self.scheme.out_dtype) + + out = self.scheme.apply_weights(layer, x) + + self.assertEqual(tuple(out.shape), (4, 12)) + torch.testing.assert_close(out, torch.matmul(x, layer.weight)) + + def test_dense_fallback_applies_bias(self): + layer, _, _ = _quantized_layer(out_features=12) + self.scheme.process_weights_after_loading(layer) + x = torch.randn(4, 32, dtype=self.scheme.out_dtype) + bias = torch.randn(12, dtype=self.scheme.out_dtype) + + biased = self.scheme.apply_weights(layer, x, bias) + + torch.testing.assert_close(biased, self.scheme.apply_weights(layer, x) + bias) + + def test_non_rocm_keeps_the_narrow_partition_quantized(self): + """The dequant is a ROCm workaround; other platforms must be untouched.""" + layer, _, _ = _quantized_layer(out_features=12) + + with patch.multiple(_MODULE, _is_hip=False, _is_fp8_fnuz=False): + with patch(f"{_MODULE}.use_aiter_bpreshuffle_gemm", return_value=False): + self.scheme.process_weights_after_loading(layer) + + self.assertEqual(layer.weight.dtype, torch.float8_e4m3fn) + self.assertIsNotNone(layer.weight_scale) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_kimi_k3_rocm_quant.py b/test/registered/unit/models/test_kimi_k3_rocm_quant.py new file mode 100644 index 0000000000..78169d1117 --- /dev/null +++ b/test/registered/unit/models/test_kimi_k3_rocm_quant.py @@ -0,0 +1,98 @@ +"""Unit tests for srt/models/kimi_k3_rocm_quant and the K3 weight-merge guard.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=12, suite="base-a-test-cpu") + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.models.kimi_k3 import _is_unquantized_mergeable +from sglang.srt.models.kimi_k3_rocm_quant import _k3_channel_fp8_to_bf16 +from sglang.test.test_utils import CustomTestCase + + +def _per_channel_fp8(out_features: int, in_features: int): + torch.manual_seed(0) + dense = torch.randn(out_features, in_features, dtype=torch.float32) + # Spread the per-channel magnitudes so a wrongly-broadcast scale cannot + # coincidentally reproduce the right answer. + dense *= torch.logspace(-1, 1, out_features).unsqueeze(1) + scale = dense.abs().amax(dim=1, keepdim=True) / 448.0 + weight = (dense / scale).to(torch.float8_e4m3fn) + return weight, scale, dense + + +class TestChannelFp8ToBf16(CustomTestCase): + """aiter's batched absorb GEMM takes ``w_scale`` as a scalar, so a + per-channel vector cannot reach it. The scale axis is the GEMM's + contraction axis, so requantizing to one per-tensor scale is lossy -- + ~2.3% relative error on real K3 weights. Dequantizing to bf16 costs ~0.12% + and lets the absorb use its bf16 path with w_scale at the 1.0 default.""" + + def test_dequantizes_every_channel_with_its_own_scale(self): + weight, scale, dense = _per_channel_fp8(out_features=8, in_features=16) + module = SimpleNamespace(weight_scale=scale.squeeze(1)) + + out = _k3_channel_fp8_to_bf16(module, weight) + + self.assertEqual(out.dtype, torch.bfloat16) + # Only the original fp8 rounding plus the bf16 cast separate this from + # the checkpoint's own values; no second quantization. + exact = weight.to(torch.float32) * scale + torch.testing.assert_close(out.float(), exact, rtol=0.01, atol=0.0) + # Guard the failure this replaced: one shared scale would leave the + # low-magnitude channels far from their true values. + worst = ((out.float() - dense).abs().sum(1) / dense.abs().sum(1)).max() + self.assertLess(worst.item(), 0.05) + + def test_accepts_a_column_vector_scale(self): + """Quark serializes the scale as [out]; callers may hold [out, 1].""" + weight, scale, _ = _per_channel_fp8(out_features=8, in_features=16) + + flat = _k3_channel_fp8_to_bf16( + SimpleNamespace(weight_scale=scale.squeeze(1)), weight + ) + column = _k3_channel_fp8_to_bf16( + SimpleNamespace(weight_scale=scale.clone()), weight + ) + + torch.testing.assert_close(flat, column) + + +class TestMergeDtypeGuard(CustomTestCase): + """``_merge_weights_as_views`` cats ``.weight`` alone, so fusing a quantized + projection silently drops the scale tensors that give its integer payload + meaning. Only dtypes that carry their full value in ``.weight`` may fuse.""" + + def test_rejects_weights_that_carry_a_separate_scale(self): + for dtype in (torch.float8_e4m3fn, torch.uint8, torch.int8): + with self.subTest(dtype=dtype): + weights = [torch.zeros(4, 4).to(dtype) for _ in range(2)] + self.assertFalse(_is_unquantized_mergeable(weights)) + + def test_accepts_plain_float_weights(self): + for dtype in (torch.bfloat16, torch.float16): + with self.subTest(dtype=dtype): + weights = [torch.zeros(4, 4, dtype=dtype) for _ in range(2)] + self.assertTrue(_is_unquantized_mergeable(weights)) + + def test_rejects_mixed_dtypes(self): + weights = [ + torch.zeros(4, 4, dtype=torch.bfloat16), + torch.zeros(4, 4, dtype=torch.float16), + ] + self.assertFalse(_is_unquantized_mergeable(weights)) + + def test_rejects_float32(self): + """The merged buffer is only built for the two half dtypes the fused + KDA/MoE kernels read; fp32 must not slip in through a dtype-agreement + check that only asks whether the operands match each other.""" + weights = [torch.zeros(4, 4, dtype=torch.float32) for _ in range(2)] + self.assertFalse(_is_unquantized_mergeable(weights)) + + +if __name__ == "__main__": + unittest.main()