[AMD][ROCm] Keep cos_sin_cache fp32 on HIP for fused QSA indexer kernel (#41282)
Co-authored-by: Nikolai Protasov <289648307+amd-nprotaso@users.noreply.github.com>
This commit is contained in:
co-authored by
Nikolai Protasov
parent
89f21671bb
commit
f6fcda8827
@@ -111,7 +111,9 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
|
||||
cache = self._compute_cos_sin_cache()
|
||||
# NOTE(ByronHsu): cache needs to be in FP32 for numerical stability.
|
||||
if not (_is_cuda or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
|
||||
# HIP: the fused QSA indexer JIT kernel (qsa_indexer.cuh) requires
|
||||
# fp32 cos_sin_cache. Keep fp32 on HIP, matching CUDA behavior.
|
||||
if not (_is_cuda or _is_hip or _is_xpu or envs.SGLANG_ROPE_CACHE_FP32.get()):
|
||||
cache = cache.to(dtype)
|
||||
|
||||
if (
|
||||
@@ -154,11 +156,19 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
|
||||
# __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`)
|
||||
# is expensive, so avoid calling it if possible
|
||||
if (
|
||||
self.cos_sin_cache.device != query.device
|
||||
or self.cos_sin_cache.dtype != query.dtype
|
||||
):
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
|
||||
if _is_hip:
|
||||
# On HIP, keep fp32 for the fused QSA indexer JIT kernel
|
||||
# (qsa_indexer.cuh requires const float* cos_sin_cache).
|
||||
if self.cos_sin_cache.device != query.device:
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
|
||||
else:
|
||||
if (
|
||||
self.cos_sin_cache.device != query.device
|
||||
or self.cos_sin_cache.dtype != query.dtype
|
||||
):
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
|
||||
"""Compute the inverse frequency."""
|
||||
@@ -436,9 +446,12 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
assert fused_set_kv_buffer_arg is None, (
|
||||
"save kv cache is not supported for fallback_rotary_embedding."
|
||||
)
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
if _is_hip:
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
|
||||
else:
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
self.fallback_rotary_embedding(
|
||||
positions,
|
||||
query,
|
||||
|
||||
@@ -23,12 +23,14 @@ from sglang.srt.runtime_context import attention_backends
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
support_triton,
|
||||
)
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
_is_cpu_amx_available = cpu_has_amx_support()
|
||||
@@ -170,11 +172,18 @@ class MRotaryEmbedding(RotaryEmbedding):
|
||||
self.position_sin = sin.repeat(1, 2).view(-1, 1, 1, last_dim).contiguous()
|
||||
|
||||
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
|
||||
if (
|
||||
self.cos_sin_cache.device != query.device
|
||||
or self.cos_sin_cache.dtype != query.dtype
|
||||
):
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype)
|
||||
if _is_hip:
|
||||
# On HIP, keep fp32 for the fused QSA indexer JIT kernel.
|
||||
if self.cos_sin_cache.device != query.device:
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(query.device)
|
||||
else:
|
||||
if (
|
||||
self.cos_sin_cache.device != query.device
|
||||
or self.cos_sin_cache.dtype != query.dtype
|
||||
):
|
||||
self.cos_sin_cache = self.cos_sin_cache.to(
|
||||
query.device, dtype=query.dtype
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user