Revert "Revert "[AMD][ROCm] Keep cos_sin_cache fp32 on HIP for fused QSA indexer kernel (#41282)"" (#42558)

This commit is contained in:
Michael
2026-10-05 16:12:21 +08:00
committed by GitHub
parent 9f7d38ffd3
commit b3f967e809
2 changed files with 36 additions and 14 deletions
@@ -111,7 +111,9 @@ class RotaryEmbedding(BaseFusedOp):
cache = self._compute_cos_sin_cache()
# NOTE(ByronHsu): cache needs to be in FP32 for numerical stability.
if not (_is_cuda or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
# HIP: the fused QSA indexer JIT kernel (qsa_indexer.cuh) requires
# fp32 cos_sin_cache. Keep fp32 on HIP, matching CUDA behavior.
if not (_is_cuda or _is_hip or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
cache = cache.to(dtype)
if (
@@ -154,11 +156,19 @@ class RotaryEmbedding(BaseFusedOp):
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
# __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`)
# is expensive, so avoid calling it if possible
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
if _is_hip:
# On HIP, keep fp32 for the fused QSA indexer JIT kernel
# (qsa_indexer.cuh requires const float* cos_sin_cache).
if self.cos_sin_cache.device != query.device:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
"""Compute the inverse frequency."""
@@ -436,9 +446,12 @@ class RotaryEmbedding(BaseFusedOp):
assert fused_set_kv_buffer_arg is None, (
"save kv cache is not supported for fallback_rotary_embedding."
)
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
if _is_hip:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
self.fallback_rotary_embedding(
positions,
query,
@@ -23,12 +23,14 @@ from sglang.srt.runtime_context import attention_backends
from sglang.srt.utils import (
cpu_has_amx_support,
is_cuda,
is_hip,
is_npu,
is_xpu,
support_triton,
)
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_npu = is_npu()
_is_xpu = is_xpu()
_is_cpu_amx_available = cpu_has_amx_support()
@@ -170,11 +172,18 @@ class MRotaryEmbedding(RotaryEmbedding):
self.position_sin = sin.repeat(1, 2).view(-1, 1, 1, last_dim).contiguous()
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
if _is_hip:
# On HIP, keep fp32 for the fused QSA indexer JIT kernel.
if self.cos_sin_cache.device != query.device:
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
else:
if (
self.cos_sin_cache.device != query.device
or self.cos_sin_cache.dtype != query.dtype
):
self.cos_sin_cache = self.cos_sin_cache.to(
query.device, dtype=query.dtype
)
def forward_native(
self,