[Fix] [AMD] qwen3: Fix KV cache write in the fused QK-norm + mRoPE decode path (#44266)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
2092160d42
commit
2f5407da28
@@ -86,6 +86,9 @@ if _is_xpu:
|
||||
class RotaryEmbedding(BaseFusedOp):
|
||||
"""Original rotary positional embedding."""
|
||||
|
||||
# (source cos_sin_cache, cast copy) memo for cos_sin_cache_as().
|
||||
_cos_sin_cache_cast: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
@@ -214,6 +217,19 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
|
||||
def cos_sin_cache_as(self, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""cos_sin_cache in ``dtype``, for kernels that read it in the activation
|
||||
dtype. The cast copy is reused until cos_sin_cache is replaced (growth or
|
||||
device move), so steady-state calls launch no cast kernel."""
|
||||
cache = self.cos_sin_cache
|
||||
if cache.dtype == dtype:
|
||||
return cache
|
||||
memo = self._cos_sin_cache_cast
|
||||
if memo is None or memo[0] is not cache or memo[1].dtype != dtype:
|
||||
memo = (cache, cache.to(dtype))
|
||||
self._cos_sin_cache_cast = memo
|
||||
return memo[1]
|
||||
|
||||
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
|
||||
"""Compute the inverse frequency."""
|
||||
# NOTE(woosuk): To exactly match the HF implementation, we need to
|
||||
|
||||
@@ -65,6 +65,65 @@ if _is_npu:
|
||||
from sglang.srt.hardware_backend.npu.cmo import get_cmo_stream, wait_cmo_stream
|
||||
|
||||
|
||||
def aiter_fused_qk_norm_rope_set_kv(
|
||||
qkv: torch.Tensor,
|
||||
q_norm: RMSNorm,
|
||||
k_norm: RMSNorm,
|
||||
rotary_emb: MRotaryEmbedding,
|
||||
positions: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""QK-norm + 3D mRoPE + KV cache write in one aiter HIP kernel.
|
||||
|
||||
``qkv`` is the (num_tokens, (num_heads + 2 * num_kv_heads) * head_dim) QKV
|
||||
projection; ``k_cache`` / ``v_cache`` are the NHD (slots, num_kv_heads,
|
||||
head_dim) pool buffers. ``k_scale`` / ``v_scale`` must be CPU tensors: the
|
||||
kernel reads them on the host. Returns q as (num_tokens, num_heads * head_dim).
|
||||
"""
|
||||
num_tokens = qkv.shape[0]
|
||||
q_out = torch.empty(
|
||||
num_tokens, num_heads, head_dim, dtype=qkv.dtype, device=qkv.device
|
||||
)
|
||||
# The kernel reads cache strides as (block, token, head); an NHD buffer is
|
||||
# one-slot blocks addressed with block_size=0.
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv.view(num_tokens, -1, head_dim),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
rotary_emb.cos_sin_cache_as(qkv.dtype),
|
||||
positions,
|
||||
num_tokens,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
rotary_emb.is_neox_style,
|
||||
rotary_emb.mrope_section,
|
||||
rotary_emb.mrope_interleaved,
|
||||
q_norm.variance_epsilon,
|
||||
q_out,
|
||||
k_cache.unsqueeze(1),
|
||||
v_cache.unsqueeze(1),
|
||||
out_cache_loc,
|
||||
k_scale,
|
||||
v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
return q_out.view(num_tokens, -1)
|
||||
|
||||
|
||||
class Qwen3Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -204,65 +263,26 @@ class Qwen3Attention(nn.Module):
|
||||
return q, k, v
|
||||
|
||||
def forward_prepare_aiter_fused_mrope(
|
||||
self, positions, hidden_states, forward_batch
|
||||
self, positions, hidden_states, forward_batch, k_cache, v_cache
|
||||
):
|
||||
"""Fused QK-norm + 3D mRoPE + KV cache write for decode (ROCm/aiter).
|
||||
|
||||
The fused HIP kernel replaces split → QK norm → mRoPE → cache write,
|
||||
so KV is already in the paged cache when this returns.
|
||||
Returns (q, None, None); caller must pass save_kv_cache=False to attn.
|
||||
"""
|
||||
"""Decode-only fused path; KV is already in the paged cache when this
|
||||
returns, so the caller must pass save_kv_cache=False to attn."""
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
num_tokens = qkv.shape[0]
|
||||
|
||||
qkv_3d = qkv.view(num_tokens, -1, self.head_dim)
|
||||
|
||||
token_to_kv_pool = get_token_to_kv_pool()
|
||||
k_cache, v_cache = token_to_kv_pool.get_kv_buffer(self.attn.layer_id)
|
||||
slot_mapping = forward_batch.out_cache_loc
|
||||
|
||||
cos_sin = self.rotary_emb.cos_sin_cache
|
||||
if cos_sin.dtype != qkv.dtype:
|
||||
cos_sin = cos_sin.to(dtype=qkv.dtype)
|
||||
|
||||
q_out = torch.empty(
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
dtype=qkv.dtype,
|
||||
device=qkv.device,
|
||||
)
|
||||
|
||||
fused_qk_norm_mrope_3d_cache_pts_quant_shuffle(
|
||||
qkv_3d,
|
||||
self.q_norm.weight,
|
||||
self.k_norm.weight,
|
||||
cos_sin,
|
||||
q = aiter_fused_qk_norm_rope_set_kv(
|
||||
qkv,
|
||||
self.q_norm,
|
||||
self.k_norm,
|
||||
self.rotary_emb,
|
||||
positions,
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self.rotary_emb.is_neox_style,
|
||||
self.rotary_emb.mrope_section,
|
||||
self.rotary_emb.mrope_interleaved,
|
||||
self.q_norm.variance_epsilon,
|
||||
q_out,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slot_mapping,
|
||||
forward_batch.out_cache_loc,
|
||||
self.num_heads,
|
||||
self.num_kv_heads,
|
||||
self.head_dim,
|
||||
self._fused_k_scale,
|
||||
self._fused_v_scale,
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
q = q_out.reshape(num_tokens, -1)
|
||||
return q, None, None
|
||||
|
||||
def forward(
|
||||
@@ -275,15 +295,23 @@ class Qwen3Attention(nn.Module):
|
||||
hidden_states = hidden_states.bfloat16()
|
||||
|
||||
save_kv_cache = True
|
||||
use_aiter_fused = (
|
||||
use_aiter_fused = False
|
||||
can_try_aiter_fused = (
|
||||
self.use_fused_qk_norm_mrope
|
||||
and forward_batch.forward_mode.is_decode()
|
||||
and get_exec().deterministic.rl_on_policy_target is None
|
||||
)
|
||||
if can_try_aiter_fused:
|
||||
k_cache, v_cache = get_token_to_kv_pool().get_kv_buffer(self.attn.layer_id)
|
||||
# The fused write targets the NHD layout in the activation dtype and
|
||||
# does not apply per-layer KV scales.
|
||||
use_aiter_fused = (
|
||||
k_cache.dim() == 3 and k_cache.dtype == hidden_states.dtype
|
||||
)
|
||||
|
||||
if use_aiter_fused:
|
||||
q, k, v = self.forward_prepare_aiter_fused_mrope(
|
||||
positions, hidden_states, forward_batch
|
||||
positions, hidden_states, forward_batch, k_cache, v_cache
|
||||
)
|
||||
save_kv_cache = False
|
||||
elif not _is_npu:
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""aiter fused QK-norm + RoPE + KV-cache write used by Qwen3 decode on ROCm.
|
||||
|
||||
``aiter_fused_qk_norm_rope_set_kv`` hands the kernel sglang's NHD
|
||||
(slots, num_kv_heads, head_dim) pool buffers. Guards: q and the K/V written at
|
||||
``out_cache_loc`` match QK-norm + RoPE computed with the reference modules, and
|
||||
no other cache slot is written.
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_amd_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_amd_ci(est_time=30, suite="stage-a-test-1-gpu-small-amd")
|
||||
|
||||
HEAD_DIM = 128
|
||||
EPS = 1e-6
|
||||
MAX_POS = 8192
|
||||
NUM_SLOTS = 4096
|
||||
# Qwen3-VL text config: interleaved 3D mRoPE over a 128-dim head.
|
||||
MROPE_SCALING = {
|
||||
"rope_type": "default",
|
||||
"mrope_section": [24, 20, 20],
|
||||
"mrope_interleaved": True,
|
||||
}
|
||||
# bf16 carries ~8 mantissa bits; q/k go through a norm and a rotation.
|
||||
TOL = 2e-2
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
torch.cuda.is_available() and torch.version.hip, "ROCm aiter kernel"
|
||||
)
|
||||
class TestQwen3AiterFusedQKNormRopeSetKV(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
os.environ.setdefault("SGLANG_USE_AITER", "1")
|
||||
from sglang.srt.models import qwen3
|
||||
|
||||
if not qwen3._has_fused_qk_norm_mrope:
|
||||
raise unittest.SkipTest("aiter fused_qk_norm_mrope kernel unavailable")
|
||||
cls.qwen3 = qwen3
|
||||
cls.device = torch.device("cuda", 0)
|
||||
|
||||
def _norm(self):
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
|
||||
norm = RMSNorm(HEAD_DIM, eps=EPS).to(self.device, torch.bfloat16)
|
||||
norm.weight.data.copy_(1.0 + 0.1 * torch.randn(HEAD_DIM, device=self.device))
|
||||
return norm
|
||||
|
||||
def _mrope(self):
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
|
||||
return get_rope(
|
||||
HEAD_DIM, HEAD_DIM, MAX_POS, 5000000, rope_scaling=MROPE_SCALING
|
||||
).to(self.device)
|
||||
|
||||
def _check(self, rotary_emb, positions, num_heads, num_kv_heads):
|
||||
num_tokens = positions.shape[-1]
|
||||
q_norm, k_norm = self._norm(), self._norm()
|
||||
qkv = torch.randn(
|
||||
num_tokens,
|
||||
(num_heads + 2 * num_kv_heads) * HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=self.device,
|
||||
)
|
||||
k_cache = torch.zeros(
|
||||
NUM_SLOTS, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device=self.device
|
||||
)
|
||||
v_cache = torch.zeros_like(k_cache)
|
||||
slots = torch.randperm(NUM_SLOTS - 1, device=self.device)[:num_tokens] + 1
|
||||
scale = torch.tensor(1.0, dtype=torch.float32, device="cpu")
|
||||
|
||||
q = self.qwen3.aiter_fused_qk_norm_rope_set_kv(
|
||||
qkv,
|
||||
q_norm,
|
||||
k_norm,
|
||||
rotary_emb,
|
||||
positions,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slots,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
scale,
|
||||
scale,
|
||||
)
|
||||
|
||||
q_ref, k_ref, v_ref = qkv.split(
|
||||
[num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM, num_kv_heads * HEAD_DIM],
|
||||
dim=-1,
|
||||
)
|
||||
q_ref = q_norm.forward_native(q_ref.reshape(-1, HEAD_DIM)).view(num_tokens, -1)
|
||||
k_ref = k_norm.forward_native(k_ref.reshape(-1, HEAD_DIM)).view(num_tokens, -1)
|
||||
q_ref, k_ref = rotary_emb.forward_native(positions, q_ref, k_ref)
|
||||
|
||||
torch.testing.assert_close(q, q_ref, atol=TOL, rtol=TOL)
|
||||
torch.testing.assert_close(
|
||||
k_cache[slots],
|
||||
k_ref.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
atol=TOL,
|
||||
rtol=TOL,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
v_cache[slots],
|
||||
v_ref.reshape(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
untouched = torch.ones(NUM_SLOTS, dtype=torch.bool, device=self.device)
|
||||
untouched[slots] = False
|
||||
self.assertFalse(k_cache[untouched].any().item(), "K written to other slots")
|
||||
self.assertFalse(v_cache[untouched].any().item(), "V written to other slots")
|
||||
|
||||
def test_mrope_nhd_cache(self):
|
||||
torch.manual_seed(0)
|
||||
rotary_emb = self._mrope()
|
||||
# (32, 8): Qwen3-VL-8B at TP1; (16, 2): Qwen3-VL-32B at TP4.
|
||||
for num_heads, num_kv_heads in ((32, 8), (16, 2)):
|
||||
for num_tokens in (1, 7, 64, 256):
|
||||
with self.subTest(heads=(num_heads, num_kv_heads), tokens=num_tokens):
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (3, num_tokens), device=self.device
|
||||
)
|
||||
self._check(rotary_emb, positions, num_heads, num_kv_heads)
|
||||
|
||||
def test_cos_sin_cache_cast_is_reused(self):
|
||||
rotary_emb = self._mrope()
|
||||
self.assertEqual(rotary_emb.cos_sin_cache.dtype, torch.float32)
|
||||
first = rotary_emb.cos_sin_cache_as(torch.bfloat16)
|
||||
self.assertIs(rotary_emb.cos_sin_cache_as(torch.bfloat16), first)
|
||||
torch.testing.assert_close(
|
||||
first, rotary_emb.cos_sin_cache.to(torch.bfloat16), atol=0, rtol=0
|
||||
)
|
||||
rotary_emb._ensure_cos_sin_cache_length(2 * MAX_POS)
|
||||
grown = rotary_emb.cos_sin_cache_as(torch.bfloat16)
|
||||
self.assertIsNot(grown, first)
|
||||
self.assertEqual(grown.shape, rotary_emb.cos_sin_cache.shape)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user