diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index 486a9d6062..d70388bf2a 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -111,7 +111,9 @@ class RotaryEmbedding(BaseFusedOp): cache = self._compute_cos_sin_cache() # NOTE(ByronHsu): cache needs to be in FP32 for numerical stability. - if not (_is_cuda or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()): + # HIP: the fused QSA indexer JIT kernel (qsa_indexer.cuh) requires + # fp32 cos_sin_cache. Keep fp32 on HIP, matching CUDA behavior. + if not (_is_cuda or _is_hip or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()): cache = cache.to(dtype) if ( @@ -154,11 +156,19 @@ class RotaryEmbedding(BaseFusedOp): def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None: # __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`) # is expensive, so avoid calling it if possible - if ( - self.cos_sin_cache.device != query.device - or self.cos_sin_cache.dtype != query.dtype - ): - self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) + if _is_hip: + # On HIP, keep fp32 for the fused QSA indexer JIT kernel + # (qsa_indexer.cuh requires const float* cos_sin_cache). + if self.cos_sin_cache.device != query.device: + self.cos_sin_cache = self.cos_sin_cache.to(query.device) + else: + if ( + self.cos_sin_cache.device != query.device + or self.cos_sin_cache.dtype != query.dtype + ): + self.cos_sin_cache = self.cos_sin_cache.to( + query.device, dtype=query.dtype + ) def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: """Compute the inverse frequency.""" @@ -436,9 +446,12 @@ class RotaryEmbedding(BaseFusedOp): assert fused_set_kv_buffer_arg is None, ( "save kv cache is not supported for fallback_rotary_embedding." ) - self.cos_sin_cache = self.cos_sin_cache.to( - query.device, dtype=query.dtype - ) + if _is_hip: + self.cos_sin_cache = self.cos_sin_cache.to(query.device) + else: + self.cos_sin_cache = self.cos_sin_cache.to( + query.device, dtype=query.dtype + ) self.fallback_rotary_embedding( positions, query, diff --git a/python/sglang/srt/layers/rotary_embedding/mrope.py b/python/sglang/srt/layers/rotary_embedding/mrope.py index 5f42f89fce..32bba1936e 100644 --- a/python/sglang/srt/layers/rotary_embedding/mrope.py +++ b/python/sglang/srt/layers/rotary_embedding/mrope.py @@ -23,12 +23,14 @@ from sglang.srt.runtime_context import attention_backends from sglang.srt.utils import ( cpu_has_amx_support, is_cuda, + is_hip, is_npu, is_xpu, support_triton, ) _is_cuda = is_cuda() +_is_hip = is_hip() _is_npu = is_npu() _is_xpu = is_xpu() _is_cpu_amx_available = cpu_has_amx_support() @@ -170,11 +172,18 @@ class MRotaryEmbedding(RotaryEmbedding): self.position_sin = sin.repeat(1, 2).view(-1, 1, 1, last_dim).contiguous() def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None: - if ( - self.cos_sin_cache.device != query.device - or self.cos_sin_cache.dtype != query.dtype - ): - self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) + if _is_hip: + # On HIP, keep fp32 for the fused QSA indexer JIT kernel. + if self.cos_sin_cache.device != query.device: + self.cos_sin_cache = self.cos_sin_cache.to(query.device) + else: + if ( + self.cos_sin_cache.device != query.device + or self.cos_sin_cache.dtype != query.dtype + ): + self.cos_sin_cache = self.cos_sin_cache.to( + query.device, dtype=query.dtype + ) def forward_native( self,