diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py index 0ad09cf0a5..0bc5390755 100755 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py @@ -21,6 +21,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.platforms.aiter import ( USE_AITER_GFX95, USE_AITER_GFX942, + USE_AITER_GFX1250, ) logger = logging.getLogger(__name__) @@ -361,8 +362,17 @@ class AITerImpl(AttentionImpl): if output is not None: return output - # BF16 path - output = aiter.flash_attn_func( + if USE_AITER_GFX1250: + return self._forward_bf16_eager(query, key, value) + return self._forward_bf16(query, key, value) + + def _forward_bf16( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> torch.Tensor: + return aiter.flash_attn_func( query, key, value, @@ -372,7 +382,17 @@ class AITerImpl(AttentionImpl): return_lse=False, how_v3_bf16_cvt=AITER_BF16_CVT_MODE, ) - return output + + # On gfx1250 flash_attn_func dispatches to a FlyDSL kernel whose Python JIT reads + # thread-local MLIR context state; Dynamo tracing into it fails at compile time. + @torch.compiler.disable + def _forward_bf16_eager( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + ) -> torch.Tensor: + return self._forward_bf16(query, key, value) @torch.compiler.disable def forward_varlen( diff --git a/python/sglang/multimodal_gen/runtime/platforms/aiter.py b/python/sglang/multimodal_gen/runtime/platforms/aiter.py index c06ef820ae..4f0333d74e 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/aiter.py +++ b/python/sglang/multimodal_gen/runtime/platforms/aiter.py @@ -4,9 +4,11 @@ from sglang.srt.utils import ( get_bool_env_var, is_gfx95_supported, is_gfx942_supported, + is_gfx1250_supported, is_hip, ) USE_AITER = get_bool_env_var("SGLANG_USE_AITER") and is_hip() USE_AITER_GFX942 = USE_AITER and is_gfx942_supported() USE_AITER_GFX95 = USE_AITER and is_gfx95_supported() +USE_AITER_GFX1250 = USE_AITER and is_gfx1250_supported() diff --git a/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py b/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py index 0745609650..18726e3928 100644 --- a/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py +++ b/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py @@ -1,7 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 """AITer attention impl construction guards (ROCm-only; skipped elsewhere).""" +import unittest +from unittest.mock import patch + import pytest +import torch HEAD_SIZE = 128 @@ -35,6 +39,34 @@ def test_rejects_kv_heads_that_do_not_divide_the_query_heads(): _build(32, 5) +class TestGfx1250BF16AttentionUnderCompile(unittest.TestCase): + """On gfx1250, flash_attn_func runs a FlyDSL kernel that JIT-compiles in Python and + fails when Dynamo traces it; a compiled DiT block must call it eagerly.""" + + def test_flash_attn_func_runs_outside_dynamo_on_gfx1250(self): + pytest.importorskip("aiter", reason="AITer is a ROCm-only dependency") + from sglang.multimodal_gen.runtime.layers.attention.backends import ( + aiter as aiter_backend, + ) + + traced_calls = [] + + def fake_flash_attn_func(q, k, v, **kwargs): + traced_calls.append(torch.compiler.is_compiling()) + return q.clone() + + impl = _build(4, 4) + q = torch.randn(1, 8, 4, HEAD_SIZE) + compiled_forward = torch.compile(impl.forward, backend="eager") + with ( + patch.object(aiter_backend, "USE_AITER_GFX1250", True), + patch.object(aiter_backend.aiter, "flash_attn_func", fake_flash_attn_func), + ): + out = compiled_forward(q, q, q) + torch.testing.assert_close(out, q) + self.assertEqual(traced_calls, [False]) + + if __name__ == "__main__": import sys