[Fix] [AMD] diffusion: Keep gfx1250 FlyDSL BF16 attention out of torch.compile (#44671)

This commit is contained in:
jacky.cheng
2026-10-19 10:36:28 -07:00
committed by GitHub
parent 4ef9765e76
commit 5ed306bd69
3 changed files with 57 additions and 3 deletions
@@ -21,6 +21,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.platforms.aiter import (
USE_AITER_GFX95,
USE_AITER_GFX942,
USE_AITER_GFX1250,
)
logger = logging.getLogger(__name__)
@@ -361,8 +362,17 @@ class AITerImpl(AttentionImpl):
if output is not None:
return output
# BF16 path
output = aiter.flash_attn_func(
if USE_AITER_GFX1250:
return self._forward_bf16_eager(query, key, value)
return self._forward_bf16(query, key, value)
def _forward_bf16(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
) -> torch.Tensor:
return aiter.flash_attn_func(
query,
key,
value,
@@ -372,7 +382,17 @@ class AITerImpl(AttentionImpl):
return_lse=False,
how_v3_bf16_cvt=AITER_BF16_CVT_MODE,
)
return output
# On gfx1250 flash_attn_func dispatches to a FlyDSL kernel whose Python JIT reads
# thread-local MLIR context state; Dynamo tracing into it fails at compile time.
@torch.compiler.disable
def _forward_bf16_eager(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
) -> torch.Tensor:
return self._forward_bf16(query, key, value)
@torch.compiler.disable
def forward_varlen(
@@ -4,9 +4,11 @@ from sglang.srt.utils import (
get_bool_env_var,
is_gfx95_supported,
is_gfx942_supported,
is_gfx1250_supported,
is_hip,
)
USE_AITER = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
USE_AITER_GFX942 = USE_AITER and is_gfx942_supported()
USE_AITER_GFX95 = USE_AITER and is_gfx95_supported()
USE_AITER_GFX1250 = USE_AITER and is_gfx1250_supported()
@@ -1,7 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
"""AITer attention impl construction guards (ROCm-only; skipped elsewhere)."""
import unittest
from unittest.mock import patch
import pytest
import torch
HEAD_SIZE = 128
@@ -35,6 +39,34 @@ def test_rejects_kv_heads_that_do_not_divide_the_query_heads():
_build(32, 5)
class TestGfx1250BF16AttentionUnderCompile(unittest.TestCase):
"""On gfx1250, flash_attn_func runs a FlyDSL kernel that JIT-compiles in Python and
fails when Dynamo traces it; a compiled DiT block must call it eagerly."""
def test_flash_attn_func_runs_outside_dynamo_on_gfx1250(self):
pytest.importorskip("aiter", reason="AITer is a ROCm-only dependency")
from sglang.multimodal_gen.runtime.layers.attention.backends import (
aiter as aiter_backend,
)
traced_calls = []
def fake_flash_attn_func(q, k, v, **kwargs):
traced_calls.append(torch.compiler.is_compiling())
return q.clone()
impl = _build(4, 4)
q = torch.randn(1, 8, 4, HEAD_SIZE)
compiled_forward = torch.compile(impl.forward, backend="eager")
with (
patch.object(aiter_backend, "USE_AITER_GFX1250", True),
patch.object(aiter_backend.aiter, "flash_attn_func", fake_flash_attn_func),
):
out = compiled_forward(q, q, q)
torch.testing.assert_close(out, q)
self.assertEqual(traced_calls, [False])
if __name__ == "__main__":
import sys