[Fix] [AMD] diffusion: Keep gfx1250 FlyDSL BF16 attention out of torch.compile (#44671)
This commit is contained in:
@@ -21,6 +21,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.platforms.aiter import (
|
||||
USE_AITER_GFX95,
|
||||
USE_AITER_GFX942,
|
||||
USE_AITER_GFX1250,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -361,8 +362,17 @@ class AITerImpl(AttentionImpl):
|
||||
if output is not None:
|
||||
return output
|
||||
|
||||
# BF16 path
|
||||
output = aiter.flash_attn_func(
|
||||
if USE_AITER_GFX1250:
|
||||
return self._forward_bf16_eager(query, key, value)
|
||||
return self._forward_bf16(query, key, value)
|
||||
|
||||
def _forward_bf16(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return aiter.flash_attn_func(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
@@ -372,7 +382,17 @@ class AITerImpl(AttentionImpl):
|
||||
return_lse=False,
|
||||
how_v3_bf16_cvt=AITER_BF16_CVT_MODE,
|
||||
)
|
||||
return output
|
||||
|
||||
# On gfx1250 flash_attn_func dispatches to a FlyDSL kernel whose Python JIT reads
|
||||
# thread-local MLIR context state; Dynamo tracing into it fails at compile time.
|
||||
@torch.compiler.disable
|
||||
def _forward_bf16_eager(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self._forward_bf16(query, key, value)
|
||||
|
||||
@torch.compiler.disable
|
||||
def forward_varlen(
|
||||
|
||||
@@ -4,9 +4,11 @@ from sglang.srt.utils import (
|
||||
get_bool_env_var,
|
||||
is_gfx95_supported,
|
||||
is_gfx942_supported,
|
||||
is_gfx1250_supported,
|
||||
is_hip,
|
||||
)
|
||||
|
||||
USE_AITER = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
USE_AITER_GFX942 = USE_AITER and is_gfx942_supported()
|
||||
USE_AITER_GFX95 = USE_AITER and is_gfx95_supported()
|
||||
USE_AITER_GFX1250 = USE_AITER and is_gfx1250_supported()
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""AITer attention impl construction guards (ROCm-only; skipped elsewhere)."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
HEAD_SIZE = 128
|
||||
|
||||
@@ -35,6 +39,34 @@ def test_rejects_kv_heads_that_do_not_divide_the_query_heads():
|
||||
_build(32, 5)
|
||||
|
||||
|
||||
class TestGfx1250BF16AttentionUnderCompile(unittest.TestCase):
|
||||
"""On gfx1250, flash_attn_func runs a FlyDSL kernel that JIT-compiles in Python and
|
||||
fails when Dynamo traces it; a compiled DiT block must call it eagerly."""
|
||||
|
||||
def test_flash_attn_func_runs_outside_dynamo_on_gfx1250(self):
|
||||
pytest.importorskip("aiter", reason="AITer is a ROCm-only dependency")
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends import (
|
||||
aiter as aiter_backend,
|
||||
)
|
||||
|
||||
traced_calls = []
|
||||
|
||||
def fake_flash_attn_func(q, k, v, **kwargs):
|
||||
traced_calls.append(torch.compiler.is_compiling())
|
||||
return q.clone()
|
||||
|
||||
impl = _build(4, 4)
|
||||
q = torch.randn(1, 8, 4, HEAD_SIZE)
|
||||
compiled_forward = torch.compile(impl.forward, backend="eager")
|
||||
with (
|
||||
patch.object(aiter_backend, "USE_AITER_GFX1250", True),
|
||||
patch.object(aiter_backend.aiter, "flash_attn_func", fake_flash_attn_func),
|
||||
):
|
||||
out = compiled_forward(q, q, q)
|
||||
torch.testing.assert_close(out, q)
|
||||
self.assertEqual(traced_calls, [False])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
|
||||
Reference in New Issue
Block a user