diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 2b3b1a8dab..299059a924 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -18,7 +18,7 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention -from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.layers.rotary_embedding import RotaryEmbedding, get_rope from sglang.srt.layers.rotary_embedding.mrope import MRotaryEmbedding from sglang.srt.layers.utils import PPMissingLayer, get_layer_id from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead @@ -50,6 +50,7 @@ _is_npu = is_npu() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip _has_fused_qk_norm_mrope = False +_has_fused_qk_norm_rope = False if _use_aiter: try: from aiter import fused_qk_norm_mrope_3d_cache_pts_quant_shuffle @@ -58,6 +59,13 @@ if _use_aiter: logger.info("aiter fused_qk_norm_mrope_3d kernel available") except ImportError: pass + try: + from aiter import fused_qk_norm_rope_cache_pts_quant_shuffle + + _has_fused_qk_norm_rope = True + logger.info("aiter fused_qk_norm_rope kernel available") + except ImportError: + pass if _is_npu: from sgl_kernel_npu.norm.split_qkv_rmsnorm_rope import split_qkv_rmsnorm_rope @@ -69,7 +77,7 @@ def aiter_fused_qk_norm_rope_set_kv( qkv: torch.Tensor, q_norm: RMSNorm, k_norm: RMSNorm, - rotary_emb: MRotaryEmbedding, + rotary_emb: RotaryEmbedding, positions: torch.Tensor, k_cache: torch.Tensor, v_cache: torch.Tensor, @@ -80,7 +88,8 @@ def aiter_fused_qk_norm_rope_set_kv( k_scale: torch.Tensor, v_scale: torch.Tensor, ) -> torch.Tensor: - """QK-norm + 3D mRoPE + KV cache write in one aiter HIP kernel. + """QK-norm + RoPE (1D, or 3D mRoPE for MRotaryEmbedding) + KV cache write in + one aiter HIP kernel. ``qkv`` is the (num_tokens, (num_heads + 2 * num_kv_heads) * head_dim) QKV projection; ``k_cache`` / ``v_cache`` are the NHD (slots, num_kv_heads, @@ -91,36 +100,42 @@ def aiter_fused_qk_norm_rope_set_kv( q_out = torch.empty( num_tokens, num_heads, head_dim, dtype=qkv.dtype, device=qkv.device ) - # The kernel reads cache strides as (block, token, head); an NHD buffer is - # one-slot blocks addressed with block_size=0. - fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( - qkv.view(num_tokens, -1, head_dim), - q_norm.weight, - k_norm.weight, - rotary_emb.cos_sin_cache_as(qkv.dtype), - positions, - num_tokens, - num_heads, - num_kv_heads, - num_kv_heads, - head_dim, - rotary_emb.is_neox_style, - rotary_emb.mrope_section, - rotary_emb.mrope_interleaved, - q_norm.variance_epsilon, - q_out, - k_cache.unsqueeze(1), - v_cache.unsqueeze(1), - out_cache_loc, - k_scale, - v_scale, - None, - None, - False, - False, - 0, - 0, + kernel_kwargs = dict( + qkv=qkv.view(num_tokens, -1, head_dim), + qw=q_norm.weight, + kw=k_norm.weight, + cos_sin=rotary_emb.cos_sin_cache_as(qkv.dtype), + positions=positions, + num_tokens=num_tokens, + num_heads_q=num_heads, + num_heads_k=num_kv_heads, + num_heads_v=num_kv_heads, + head_size=head_dim, + is_neox_style=rotary_emb.is_neox_style, + eps=q_norm.variance_epsilon, + q_out=q_out, + # The kernel reads cache strides as (block, token, head); an NHD buffer + # is one-slot blocks addressed with block_size=0. + k_cache=k_cache.unsqueeze(1), + v_cache=v_cache.unsqueeze(1), + slot_mapping=out_cache_loc, + per_tensor_k_scale=k_scale, + per_tensor_v_scale=v_scale, + k_out=None, + v_out=None, + return_kv=False, + use_shuffle_layout=False, + block_size=0, + x=0, ) + if isinstance(rotary_emb, MRotaryEmbedding): + fused_qk_norm_mrope_3d_cache_pts_quant_shuffle( + **kernel_kwargs, + mrope_section_=rotary_emb.mrope_section, + is_interleaved=rotary_emb.mrope_interleaved, + ) + else: + fused_qk_norm_rope_cache_pts_quant_shuffle(**kernel_kwargs) return q_out.view(num_tokens, -1) @@ -220,7 +235,17 @@ class Qwen3Attention(nn.Module): and isinstance(self.rotary_emb, MRotaryEmbedding) and getattr(self.rotary_emb, "mrope_section", None) is not None ) - if self.use_fused_qk_norm_mrope: + # The 1D kernel is instantiated for these head sizes, indexes + # cos_sin_cache as [pos, rotary_dim] and takes one eps for both norms; + # scaled-RoPE subclasses stay on the unfused path. + self.use_fused_qk_norm_rope = ( + _has_fused_qk_norm_rope + and type(self.rotary_emb) is RotaryEmbedding + and self.rotary_emb.rotary_dim == self.head_dim + and self.head_dim in (64, 128, 256, 512) + and self.q_norm.variance_epsilon == self.k_norm.variance_epsilon + ) + if self.use_fused_qk_norm_mrope or self.use_fused_qk_norm_rope: # Scale tensors MUST stay on CPU: the C++ kernel uses .item() # which triggers hipMemcpy D2H + sync on CUDA tensors, breaking graph capture. # Explicit device='cpu' is required because SGLang constructs models inside @@ -262,7 +287,7 @@ class Qwen3Attention(nn.Module): ) return q, k, v - def forward_prepare_aiter_fused_mrope( + def forward_prepare_aiter_fused( self, positions, hidden_states, forward_batch, k_cache, v_cache ): """Decode-only fused path; KV is already in the paged cache when this @@ -297,7 +322,7 @@ class Qwen3Attention(nn.Module): save_kv_cache = True use_aiter_fused = False can_try_aiter_fused = ( - self.use_fused_qk_norm_mrope + (self.use_fused_qk_norm_mrope or self.use_fused_qk_norm_rope) and forward_batch.forward_mode.is_decode() and get_exec().deterministic.rl_on_policy_target is None ) @@ -310,7 +335,7 @@ class Qwen3Attention(nn.Module): ) if use_aiter_fused: - q, k, v = self.forward_prepare_aiter_fused_mrope( + q, k, v = self.forward_prepare_aiter_fused( positions, hidden_states, forward_batch, k_cache, v_cache ) save_kv_cache = False diff --git a/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py b/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py index 8f9a0cdc68..1645dc59cc 100644 --- a/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py +++ b/test/registered/amd/test_qwen3_aiter_fused_qk_norm_rope.py @@ -39,8 +39,8 @@ class TestQwen3AiterFusedQKNormRopeSetKV(CustomTestCase): os.environ.setdefault("SGLANG_USE_AITER", "1") from sglang.srt.models import qwen3 - if not qwen3._has_fused_qk_norm_mrope: - raise unittest.SkipTest("aiter fused_qk_norm_mrope kernel unavailable") + if not (qwen3._has_fused_qk_norm_mrope or qwen3._has_fused_qk_norm_rope): + raise unittest.SkipTest("aiter fused QK-norm + RoPE kernels unavailable") cls.qwen3 = qwen3 cls.device = torch.device("cuda", 0) @@ -51,6 +51,11 @@ class TestQwen3AiterFusedQKNormRopeSetKV(CustomTestCase): norm.weight.data.copy_(1.0 + 0.1 * torch.randn(HEAD_DIM, device=self.device)) return norm + def _rope(self): + from sglang.srt.layers.rotary_embedding import get_rope + + return get_rope(HEAD_DIM, HEAD_DIM, MAX_POS, 1000000).to(self.device) + def _mrope(self): from sglang.srt.layers.rotary_embedding import get_rope @@ -116,7 +121,23 @@ class TestQwen3AiterFusedQKNormRopeSetKV(CustomTestCase): self.assertFalse(k_cache[untouched].any().item(), "K written to other slots") self.assertFalse(v_cache[untouched].any().item(), "V written to other slots") + def test_rope_nhd_cache(self): + if not self.qwen3._has_fused_qk_norm_rope: + self.skipTest("aiter fused_qk_norm_rope kernel unavailable") + torch.manual_seed(0) + rotary_emb = self._rope() + # (32, 8): Qwen3-8B at TP1; (8, 2): Qwen3-8B at TP4. + for num_heads, num_kv_heads in ((32, 8), (8, 2)): + for num_tokens in (1, 7, 64, 256): + with self.subTest(heads=(num_heads, num_kv_heads), tokens=num_tokens): + positions = torch.randint( + 0, MAX_POS, (num_tokens,), device=self.device + ) + self._check(rotary_emb, positions, num_heads, num_kv_heads) + def test_mrope_nhd_cache(self): + if not self.qwen3._has_fused_qk_norm_mrope: + self.skipTest("aiter fused_qk_norm_mrope kernel unavailable") torch.manual_seed(0) rotary_emb = self._mrope() # (32, 8): Qwen3-VL-8B at TP1; (16, 2): Qwen3-VL-32B at TP4.