From 2f5407da286bebd645440b5fe2293bd6e4fbfcc5 Mon Sep 17 00:00:00 2001 From: jiaqiang-dot-liu Date: Thu, 15 Oct 2026 21:22:46 +0800 Subject: [PATCH] [Fix] [AMD] qwen3: Fix KV cache write in the fused QK-norm + mRoPE decode path (#44266) Co-authored-by: Cursor --- .../srt/layers/rotary_embedding/base.py | 16 ++ python/sglang/srt/models/qwen3.py | 134 +++++++++------- .../test_qwen3_aiter_fused_qk_norm_rope.py | 146 ++++++++++++++++++ 3 files changed, 243 insertions(+), 53 deletions(-) create mode 100644 test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index 2a9a74992b..20a9425958 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -86,6 +86,9 @@ if _is_xpu: class RotaryEmbedding(BaseFusedOp): """Original rotary positional embedding.""" + # (source cos_sin_cache, cast copy) memo for cos_sin_cache_as(). + _cos_sin_cache_cast: Optional[Tuple[torch.Tensor, torch.Tensor]] = None + def __init__( self, head_size: int, @@ -214,6 +217,19 @@ class RotaryEmbedding(BaseFusedOp): query.device, dtype=query.dtype ) + def cos_sin_cache_as(self, dtype: torch.dtype) -> torch.Tensor: + """cos_sin_cache in ``dtype``, for kernels that read it in the activation + dtype. The cast copy is reused until cos_sin_cache is replaced (growth or + device move), so steady-state calls launch no cast kernel.""" + cache = self.cos_sin_cache + if cache.dtype == dtype: + return cache + memo = self._cos_sin_cache_cast + if memo is None or memo[0] is not cache or memo[1].dtype != dtype: + memo = (cache, cache.to(dtype)) + self._cos_sin_cache_cast = memo + return memo[1] + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: """Compute the inverse frequency.""" # NOTE(woosuk): To exactly match the HF implementation, we need to diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 9552a15ba7..2b3b1a8dab 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -65,6 +65,65 @@ if _is_npu: from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream +def aiter_fused_qk_norm_rope_set_kv( + qkv: torch.Tensor, + q_norm: RMSNorm, + k_norm: RMSNorm, + rotary_emb: MRotaryEmbedding, + positions: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + out_cache_loc: torch.Tensor, + num_heads: int, + num_kv_heads: int, + head_dim: int, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> torch.Tensor: + """QK-norm + 3D mRoPE + KV cache write in one aiter HIP kernel. + + ``qkv`` is the (num_tokens, (num_heads + 2 * num_kv_heads) * head_dim) QKV + projection; ``k_cache`` / ``v_cache`` are the NHD (slots, num_kv_heads, + head_dim) pool buffers. ``k_scale`` / ``v_scale`` must be CPU tensors: the + kernel reads them on the host. Returns q as (num_tokens, num_heads * head_dim). + """ + num_tokens = qkv.shape[0] + q_out = torch.empty( + num_tokens, num_heads, head_dim, dtype=qkv.dtype, device=qkv.device + ) + # The kernel reads cache strides as (block, token, head); an NHD buffer is + # one-slot blocks addressed with block_size=0. + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( + qkv.view(num_tokens, -1, head_dim), + q_norm.weight, + k_norm.weight, + rotary_emb.cos_sin_cache_as(qkv.dtype), + positions, + num_tokens, + num_heads, + num_kv_heads, + num_kv_heads, + head_dim, + rotary_emb.is_neox_style, + rotary_emb.mrope_section, + rotary_emb.mrope_interleaved, + q_norm.variance_epsilon, + q_out, + k_cache.unsqueeze(1), + v_cache.unsqueeze(1), + out_cache_loc, + k_scale, + v_scale, + None, + None, + False, + False, + 0, + 0, + ) + return q_out.view(num_tokens, -1) + + class Qwen3Attention(nn.Module): def __init__( self, @@ -204,65 +263,26 @@ class Qwen3Attention(nn.Module): return q, k, v def forward_prepare_aiter_fused_mrope( - self, positions, hidden_states, forward_batch + self, positions, hidden_states, forward_batch, k_cache, v_cache ): - """Fused QK-norm + 3D mRoPE + KV cache write for decode (ROCm/aiter). - - The fused HIP kernel replaces split → QK norm → mRoPE → cache write, - so KV is already in the paged cache when this returns. - Returns (q, None, None); caller must pass save_kv_cache=False to attn. - """ + """Decode-only fused path; KV is already in the paged cache when this + returns, so the caller must pass save_kv_cache=False to attn.""" qkv, _ = self.qkv_proj(hidden_states) - num_tokens = qkv.shape[0] - - qkv_3d = qkv.view(num_tokens, -1, self.head_dim) - - token_to_kv_pool = get_token_to_kv_pool() - k_cache, v_cache = token_to_kv_pool.get_kv_buffer(self.attn.layer_id) - slot_mapping = forward_batch.out_cache_loc - - cos_sin = self.rotary_emb.cos_sin_cache - if cos_sin.dtype != qkv.dtype: - cos_sin = cos_sin.to(dtype=qkv.dtype) - - q_out = torch.empty( - num_tokens, - self.num_heads, - self.head_dim, - dtype=qkv.dtype, - device=qkv.device, - ) - - fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( - qkv_3d, - self.q_norm.weight, - self.k_norm.weight, - cos_sin, + q = aiter_fused_qk_norm_rope_set_kv( + qkv, + self.q_norm, + self.k_norm, + self.rotary_emb, positions, - num_tokens, - self.num_heads, - self.num_kv_heads, - self.num_kv_heads, - self.head_dim, - self.rotary_emb.is_neox_style, - self.rotary_emb.mrope_section, - self.rotary_emb.mrope_interleaved, - self.q_norm.variance_epsilon, - q_out, k_cache, v_cache, - slot_mapping, + forward_batch.out_cache_loc, + self.num_heads, + self.num_kv_heads, + self.head_dim, self._fused_k_scale, self._fused_v_scale, - None, - None, - False, - False, - 0, - 0, ) - - q = q_out.reshape(num_tokens, -1) return q, None, None def forward( @@ -275,15 +295,23 @@ class Qwen3Attention(nn.Module): hidden_states = hidden_states.bfloat16() save_kv_cache = True - use_aiter_fused = ( + use_aiter_fused = False + can_try_aiter_fused = ( self.use_fused_qk_norm_mrope and forward_batch.forward_mode.is_decode() and get_exec().deterministic.rl_on_policy_target is None ) + if can_try_aiter_fused: + k_cache, v_cache = get_token_to_kv_pool().get_kv_buffer(self.attn.layer_id) + # The fused write targets the NHD layout in the activation dtype and + # does not apply per-layer KV scales. + use_aiter_fused = ( + k_cache.dim() == 3 and k_cache.dtype == hidden_states.dtype + ) if use_aiter_fused: q, k, v = self.forward_prepare_aiter_fused_mrope( - positions, hidden_states, forward_batch + positions, hidden_states, forward_batch, k_cache, v_cache ) save_kv_cache = False elif not _is_npu: diff --git a/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py b/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py new file mode 100644 index 0000000000..8f9a0cdc68 --- /dev/null +++ b/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py @@ -0,0 +1,146 @@ +"""aiter fused QK-norm + RoPE + KV-cache write used by Qwen3 decode on ROCm. + +``aiter_fused_qk_norm_rope_set_kv`` hands the kernel sglang's NHD +(slots, num_kv_heads, head_dim) pool buffers. Guards: q and the K/V written at +``out_cache_loc`` match QK-norm + RoPE computed with the reference modules, and +no other cache slot is written. +""" + +import os +import unittest + +import torch + +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=30, suite="stage-a-test-1-gpu-small-amd") + +HEAD_DIM = 128 +EPS = 1e-6 +MAX_POS = 8192 +NUM_SLOTS = 4096 +# Qwen3-VL text config: interleaved 3D mRoPE over a 128-dim head. +MROPE_SCALING = { + "rope_type": "default", + "mrope_section": [24, 20, 20], + "mrope_interleaved": True, +} +# bf16 carries ~8 mantissa bits; q/k go through a norm and a rotation. +TOL = 2e-2 + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.version.hip, "ROCm aiter kernel" +) +class TestQwen3AiterFusedQKNormRopeSetKV(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ.setdefault("SGLANG_USE_AITER", "1") + from sglang.srt.models import qwen3 + + if not qwen3._has_fused_qk_norm_mrope: + raise unittest.SkipTest("aiter fused_qk_norm_mrope kernel unavailable") + cls.qwen3 = qwen3 + cls.device = torch.device("cuda", 0) + + def _norm(self): + from sglang.srt.layers.layernorm import RMSNorm + + norm = RMSNorm(HEAD_DIM, eps=EPS).to(self.device, torch.bfloat16) + norm.weight.data.copy_(1.0 + 0.1 * torch.randn(HEAD_DIM, device=self.device)) + return norm + + def _mrope(self): + from sglang.srt.layers.rotary_embedding import get_rope + + return get_rope( + HEAD_DIM, HEAD_DIM, MAX_POS, 5000000, rope_scaling=MROPE_SCALING + ).to(self.device) + + def _check(self, rotary_emb, positions, num_heads, num_kv_heads): + num_tokens = positions.shape[-1] + q_norm, k_norm = self._norm(), self._norm() + qkv = torch.randn( + num_tokens, + (num_heads + 2 * num_kv_heads) * HEAD_DIM, + dtype=torch.bfloat16, + device=self.device, + ) + k_cache = torch.zeros( + NUM_SLOTS, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device=self.device + ) + v_cache = torch.zeros_like(k_cache) + slots = torch.randperm(NUM_SLOTS - 1, device=self.device)[:num_tokens] + 1 + scale = torch.tensor(1.0, dtype=torch.float32, device="cpu") + + q = self.qwen3.aiter_fused_qk_norm_rope_set_kv( + qkv, + q_norm, + k_norm, + rotary_emb, + positions, + k_cache, + v_cache, + slots, + num_heads, + num_kv_heads, + HEAD_DIM, + scale, + scale, + ) + + q_ref, k_ref, v_ref = qkv.split( + [num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM, num_kv_heads * HEAD_DIM], + dim=-1, + ) + q_ref = q_norm.forward_native(q_ref.reshape(-1, HEAD_DIM)).view(num_tokens, -1) + k_ref = k_norm.forward_native(k_ref.reshape(-1, HEAD_DIM)).view(num_tokens, -1) + q_ref, k_ref = rotary_emb.forward_native(positions, q_ref, k_ref) + + torch.testing.assert_close(q, q_ref, atol=TOL, rtol=TOL) + torch.testing.assert_close( + k_cache[slots], + k_ref.view(num_tokens, num_kv_heads, HEAD_DIM), + atol=TOL, + rtol=TOL, + ) + torch.testing.assert_close( + v_cache[slots], + v_ref.reshape(num_tokens, num_kv_heads, HEAD_DIM), + atol=0, + rtol=0, + ) + untouched = torch.ones(NUM_SLOTS, dtype=torch.bool, device=self.device) + untouched[slots] = False + self.assertFalse(k_cache[untouched].any().item(), "K written to other slots") + self.assertFalse(v_cache[untouched].any().item(), "V written to other slots") + + def test_mrope_nhd_cache(self): + torch.manual_seed(0) + rotary_emb = self._mrope() + # (32, 8): Qwen3-VL-8B at TP1; (16, 2): Qwen3-VL-32B at TP4. + for num_heads, num_kv_heads in ((32, 8), (16, 2)): + for num_tokens in (1, 7, 64, 256): + with self.subTest(heads=(num_heads, num_kv_heads), tokens=num_tokens): + positions = torch.randint( + 0, MAX_POS, (3, num_tokens), device=self.device + ) + self._check(rotary_emb, positions, num_heads, num_kv_heads) + + def test_cos_sin_cache_cast_is_reused(self): + rotary_emb = self._mrope() + self.assertEqual(rotary_emb.cos_sin_cache.dtype, torch.float32) + first = rotary_emb.cos_sin_cache_as(torch.bfloat16) + self.assertIs(rotary_emb.cos_sin_cache_as(torch.bfloat16), first) + torch.testing.assert_close( + first, rotary_emb.cos_sin_cache.to(torch.bfloat16), atol=0, rtol=0 + ) + rotary_emb._ensure_cos_sin_cache_length(2 * MAX_POS) + grown = rotary_emb.cos_sin_cache_as(torch.bfloat16) + self.assertIsNot(grown, first) + self.assertEqual(grown.shape, rotary_emb.cos_sin_cache.shape) + + +if __name__ == "__main__": + unittest.main()