From 88ac7c02a68c83b3c92fbd70bbb9d9262e56a92e Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Sat, 24 Oct 2026 10:42:54 +0800 Subject: [PATCH] [Fix] [AMD] diffusion: Fix the first-step denoise regression from compiling ENABLE_CK=0 BF16 attention (#45838) --- .../layers/attention/backends/aiter.py | 5 ++-- .../test/unit/test_aiter_attention_impl.py | 27 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py index 0bc5390755..82dca1d29e 100755 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/aiter.py @@ -10,6 +10,7 @@ import aiter import torch import triton import triton.language as tl +from aiter.jit.core import ENABLE_CK as _AITER_ENABLE_CK from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( AttentionBackend, @@ -362,7 +363,7 @@ class AITerImpl(AttentionImpl): if output is not None: return output - if USE_AITER_GFX1250: + if USE_AITER_GFX1250 or not _AITER_ENABLE_CK: return self._forward_bf16_eager(query, key, value) return self._forward_bf16(query, key, value) @@ -383,8 +384,6 @@ class AITerImpl(AttentionImpl): how_v3_bf16_cvt=AITER_BF16_CVT_MODE, ) - # On gfx1250 flash_attn_func dispatches to a FlyDSL kernel whose Python JIT reads - # thread-local MLIR context state; Dynamo tracing into it fails at compile time. @torch.compiler.disable def _forward_bf16_eager( self, diff --git a/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py b/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py index 18726e3928..7594580988 100644 --- a/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py +++ b/python/sglang/multimodal_gen/test/unit/test_aiter_attention_impl.py @@ -39,16 +39,21 @@ def test_rejects_kv_heads_that_do_not_divide_the_query_heads(): _build(32, 5) -class TestGfx1250BF16AttentionUnderCompile(unittest.TestCase): - """On gfx1250, flash_attn_func runs a FlyDSL kernel that JIT-compiles in Python and - fails when Dynamo traces it; a compiled DiT block must call it eagerly.""" +class TestBF16AttentionUnderCompile(unittest.TestCase): + """BF16 flash_attn_func must run outside Dynamo when tracing it would skip the DiT. - def test_flash_attn_func_runs_outside_dynamo_on_gfx1250(self): + gfx1250 dispatches to a FlyDSL kernel that JIT-compiles in Python. ENABLE_CK=0 + dispatches to aiter's Triton kernel, which opens a config JSON. Either graph + break sits inside the block loop and drops the compiled forward. + """ + + def _compiled_call(self, *, gfx1250: bool, enable_ck: bool) -> list[bool]: pytest.importorskip("aiter", reason="AITer is a ROCm-only dependency") from sglang.multimodal_gen.runtime.layers.attention.backends import ( aiter as aiter_backend, ) + torch._dynamo.reset() traced_calls = [] def fake_flash_attn_func(q, k, v, **kwargs): @@ -59,12 +64,22 @@ class TestGfx1250BF16AttentionUnderCompile(unittest.TestCase): q = torch.randn(1, 8, 4, HEAD_SIZE) compiled_forward = torch.compile(impl.forward, backend="eager") with ( - patch.object(aiter_backend, "USE_AITER_GFX1250", True), + patch.object(aiter_backend, "USE_AITER_GFX1250", gfx1250), + patch.object(aiter_backend, "_AITER_ENABLE_CK", enable_ck), patch.object(aiter_backend.aiter, "flash_attn_func", fake_flash_attn_func), ): out = compiled_forward(q, q, q) torch.testing.assert_close(out, q) - self.assertEqual(traced_calls, [False]) + return traced_calls + + def test_flash_attn_func_runs_outside_dynamo_on_gfx1250(self): + self.assertEqual(self._compiled_call(gfx1250=True, enable_ck=True), [False]) + + def test_flash_attn_func_runs_outside_dynamo_when_ck_is_disabled(self): + self.assertEqual(self._compiled_call(gfx1250=False, enable_ck=False), [False]) + + def test_ck_flash_attn_stays_inside_the_compiled_graph(self): + self.assertEqual(self._compiled_call(gfx1250=False, enable_ck=True), [True]) if __name__ == "__main__":