diff --git a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py index 5643dbae16..864bc3c27a 100644 --- a/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py +++ b/python/sglang/srt/layers/attention/qwen_sparse_attn_backend.py @@ -72,7 +72,7 @@ def _resolve_trtllm_sparse_decode(): @lru_cache(maxsize=1) def _resolve_flash_attn_varlen_func(): - from sglang.srt.utils import is_sm121 + from sglang.srt.utils import is_hip, is_sm121 if is_sm121(): from sglang.kernels.ops.attention import ( @@ -80,6 +80,22 @@ def _resolve_flash_attn_varlen_func(): ) return qwen38_qsa_sm121_varlen + if is_hip(): + # ROCm ships neither flash_attn (FA2) nor flash-attn-4; aiter provides an + # FA2-compatible varlen kernel (CK) with the same call signature. Fall + # through to the flash_attn probes below if aiter is unavailable. + try: + from aiter import flash_attn_varlen_func as aiter_varlen_func + except ImportError: + aiter_varlen_func = None + if aiter_varlen_func is not None: + + def flash_attn_varlen_func(*args, **kwargs): + output = aiter_varlen_func(*args, **kwargs) + # aiter returns a tuple when return_lse/return_attn_probs is set. + return output[0] if isinstance(output, tuple) else output + + return flash_attn_varlen_func try: from flash_attn import flash_attn_varlen_func @@ -97,8 +113,8 @@ def _resolve_flash_attn_varlen_func(): return flash_attn_varlen_func except ImportError as exc: raise ImportError( - "QSA decode requires flash_attn (FA2) or flash-attn-4 " - "(FA4 cute) for its packed varlen fallback." + "QSA decode requires flash_attn (FA2), flash-attn-4 (FA4 cute), " + "or aiter (ROCm) for its packed varlen fallback." ) from exc diff --git a/test/registered/kernels/ops/attention/qsa/test_qsa_rocm.py b/test/registered/kernels/ops/attention/qsa/test_qsa_rocm.py new file mode 100644 index 0000000000..d15edee630 --- /dev/null +++ b/test/registered/kernels/ops/attention/qsa/test_qsa_rocm.py @@ -0,0 +1,125 @@ +"""ROCm coverage for the QSA packed-varlen decode fallback. + +On HIP there is neither flash_attn (FA2) nor flash-attn-4, so +``_resolve_flash_attn_varlen_func`` resolves to aiter's FA2-compatible varlen +kernel. The first two tests are device-independent (fake modules); the last one +runs the real aiter kernel against a torch reference and only executes on ROCm. +""" + +import sys +from types import ModuleType + +import pytest +import torch + +from sglang.srt.layers.attention import qwen_sparse_attn_backend as qsa_backend_module +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-small") +register_amd_ci(est_time=60, stage="jit-kernel-unit", runner_config="amd") + + +def _fake_aiter(varlen_func): + module = ModuleType("aiter") + module.flash_attn_varlen_func = varlen_func + return module + + +@pytest.mark.parametrize("returns_tuple", [False, True], ids=["tensor", "tuple"]) +def test_qsa_hip_resolves_aiter_varlen_kernel(monkeypatch, returns_tuple): + resolver = qsa_backend_module._resolve_flash_attn_varlen_func + resolver.cache_clear() + sentinel = object() + calls = [] + + def aiter_varlen(*args, **kwargs): + calls.append(kwargs) + return (sentinel, None) if returns_tuple else sentinel + + monkeypatch.setattr("sglang.srt.utils.is_sm121", lambda: False) + monkeypatch.setattr("sglang.srt.utils.is_hip", lambda: True) + monkeypatch.setitem(sys.modules, "aiter", _fake_aiter(aiter_varlen)) + + try: + func = resolver() + assert func(q=1, causal=True) is sentinel + assert calls == [{"q": 1, "causal": True}] + finally: + resolver.cache_clear() + + +def test_qsa_hip_without_aiter_falls_through_to_flash_attn(monkeypatch): + resolver = qsa_backend_module._resolve_flash_attn_varlen_func + resolver.cache_clear() + fa2_func = object() + flash_attn = ModuleType("flash_attn") + flash_attn.flash_attn_varlen_func = fa2_func + + monkeypatch.setattr("sglang.srt.utils.is_sm121", lambda: False) + monkeypatch.setattr("sglang.srt.utils.is_hip", lambda: True) + monkeypatch.setitem(sys.modules, "aiter", None) # makes `import aiter` fail + monkeypatch.setitem(sys.modules, "flash_attn", flash_attn) + + try: + assert resolver() is fa2_func + finally: + resolver.cache_clear() + + +def test_qsa_hip_aiter_varlen_matches_torch_reference(): + if not (is_hip() and torch.cuda.is_available()): + pytest.skip("ROCm-only kernel") + pytest.importorskip("aiter") + + resolver = qsa_backend_module._resolve_flash_attn_varlen_func + resolver.cache_clear() + torch.manual_seed(2028) + device = torch.device("cuda") + # Qwen3.8-Flash-Next full-attention shape: 24 query heads, 2 KV heads, d=256. + batch, topk = 3, 2048 + num_q_heads, num_kv_heads, head_dim = 24, 2, 256 + valid_counts = torch.tensor([5, topk, 700], dtype=torch.int32, device=device) + cu_seqlens_k = torch.zeros(batch + 1, dtype=torch.int32, device=device) + cu_seqlens_k[1:] = torch.cumsum(valid_counts, 0) + cu_seqlens_q = torch.arange(batch + 1, dtype=torch.int32, device=device) + q = torch.randn(batch, num_q_heads, head_dim, device=device, dtype=torch.bfloat16) + packed_k = torch.randn( + int(valid_counts.sum()), + num_kv_heads, + head_dim, + device=device, + dtype=torch.bfloat16, + ) + packed_v = torch.randn_like(packed_k) + scale = head_dim**-0.5 + + try: + # Mirrors the decode call in QwenSparseAttnBackend._forward_paged_attention. + out = resolver()( + q=q, + k=packed_k, + v=packed_v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=1, + max_seqlen_k=topk, + softmax_scale=scale, + causal=True, + ) + finally: + resolver.cache_clear() + + assert out.shape == (batch, num_q_heads, head_dim) + repeats = num_q_heads // num_kv_heads + for row in range(batch): + start, end = int(cu_seqlens_k[row]), int(cu_seqlens_k[row + 1]) + keys = packed_k[start:end].float().repeat_interleave(repeats, dim=1) + values = packed_v[start:end].float().repeat_interleave(repeats, dim=1) + scores = torch.einsum("hd,khd->hk", q[row].float(), keys) * scale + expected = torch.einsum("hk,khd->hd", scores.softmax(dim=-1), values) + torch.testing.assert_close(out[row].float(), expected, atol=2e-2, rtol=2e-2) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))