[AMD][ROCm] Keep cos_sin_cache fp32 on HIP for fused QSA indexer kernel (#41282)

Co-authored-by: Nikolai Protasov <289648307+amd-nprotaso@users.noreply.github.com>
This commit is contained in:
ChangLiu0709
2026-10-02 11:57:03 -07:00
committed by GitHub
co-authored by Nikolai Protasov
parent 89f21671bb
commit f6fcda8827
2 changed files with 36 additions and 14 deletions
@@ -111,7 +111,9 @@ class RotaryEmbedding(BaseFusedOp):
cache = self._compute_cos_sin_cache()
# NOTE(ByronHsu): cache needs to be in FP32 for numerical stability.
if not (_is_cuda or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
# HIP: the fused QSA indexer JIT kernel (qsa_indexer.cuh) requires
# fp32 cos_sin_cache. Keep fp32 on HIP, matching CUDA behavior.
if not (_is_cuda or _is_hip or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
cache = cache.to(dtype)
if (
@@ -154,11 +156,19 @@ class RotaryEmbedding(BaseFusedOp):
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
# __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`)
# is expensive, so avoid calling it if possible
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
if _is_hip:
# On HIP, keep fp32 for the fused QSA indexer JIT kernel
# (qsa_indexer.cuh requires const float* cos_sin_cache).
if self.cos_sin_cache.device != query.device:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
"""Compute the inverse frequency."""
@@ -436,9 +446,12 @@ class RotaryEmbedding(BaseFusedOp):
assert fused_set_kv_buffer_arg is None, (
"save kv cache is not supported for fallback_rotary_embedding."
)
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
if _is_hip:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
self.fallback_rotary_embedding(
positions,
query,
@@ -23,12 +23,14 @@ from sglang.srt.runtime_context import attention_backends
from sglang.srt.utils import (
cpu_has_amx_support,
is_cuda,
is_hip,
is_npu,
is_xpu,
support_triton,
)
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
_is_cpu_amx_available = cpu_has_amx_support()
@@ -170,11 +172,18 @@ class MRotaryEmbedding(RotaryEmbedding):
self.position_sin = sin.repeat(1, 2).view(-1, 1, 1, last_dim).contiguous()
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
if _is_hip:
# On HIP, keep fp32 for the fused QSA indexer JIT kernel.
if self.cos_sin_cache.device != query.device:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
def forward_native(
self,