diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index d70388bf2a..486a9d6062 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -111,9 +111,7 @@ class RotaryEmbedding(BaseFusedOp): cache = self._compute_cos_sin_cache() # NOTE(ByronHsu): cache needs to be in FP32 for numerical stability. - # HIP: the fused QSA indexer JIT kernel (qsa_indexer.cuh) requires - # fp32 cos_sin_cache. Keep fp32 on HIP, matching CUDA behavior. - if not (_is_cuda or _is_hip or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()): + if not (_is_cuda or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()): cache = cache.to(dtype) if ( @@ -156,19 +154,11 @@ class RotaryEmbedding(BaseFusedOp): def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None: # __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`) # is expensive, so avoid calling it if possible - if _is_hip: - # On HIP, keep fp32 for the fused QSA indexer JIT kernel - # (qsa_indexer.cuh requires const float* cos_sin_cache). - if self.cos_sin_cache.device != query.device: - self.cos_sin_cache = self.cos_sin_cache.to(query.device) - else: - if ( - self.cos_sin_cache.device != query.device - or self.cos_sin_cache.dtype != query.dtype - ): - self.cos_sin_cache = self.cos_sin_cache.to( - query.device, dtype=query.dtype - ) + if ( + self.cos_sin_cache.device != query.device + or self.cos_sin_cache.dtype != query.dtype + ): + self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: """Compute the inverse frequency.""" @@ -446,12 +436,9 @@ class RotaryEmbedding(BaseFusedOp): assert fused_set_kv_buffer_arg is None, ( "save kv cache is not supported for fallback_rotary_embedding." ) - if _is_hip: - self.cos_sin_cache = self.cos_sin_cache.to(query.device) - else: - self.cos_sin_cache = self.cos_sin_cache.to( - query.device, dtype=query.dtype - ) + self.cos_sin_cache = self.cos_sin_cache.to( + query.device, dtype=query.dtype + ) self.fallback_rotary_embedding( positions, query, diff --git a/python/sglang/srt/layers/rotary_embedding/mrope.py b/python/sglang/srt/layers/rotary_embedding/mrope.py index 32bba1936e..5f42f89fce 100644 --- a/python/sglang/srt/layers/rotary_embedding/mrope.py +++ b/python/sglang/srt/layers/rotary_embedding/mrope.py @@ -23,14 +23,12 @@ from sglang.srt.runtime_context import attention_backends from sglang.srt.utils import ( cpu_has_amx_support, is_cuda, - is_hip, is_npu, is_xpu, support_triton, ) _is_cuda = is_cuda() -_is_hip = is_hip() _is_npu = is_npu() _is_xpu = is_xpu() _is_cpu_amx_available = cpu_has_amx_support() @@ -172,18 +170,11 @@ class MRotaryEmbedding(RotaryEmbedding): self.position_sin = sin.repeat(1, 2).view(-1, 1, 1, last_dim).contiguous() def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None: - if _is_hip: - # On HIP, keep fp32 for the fused QSA indexer JIT kernel. - if self.cos_sin_cache.device != query.device: - self.cos_sin_cache = self.cos_sin_cache.to(query.device) - else: - if ( - self.cos_sin_cache.device != query.device - or self.cos_sin_cache.dtype != query.dtype - ): - self.cos_sin_cache = self.cos_sin_cache.to( - query.device, dtype=query.dtype - ) + if ( + self.cos_sin_cache.device != query.device + or self.cos_sin_cache.dtype != query.dtype + ): + self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) def forward_native( self,