diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py index ed688c8ddf..654fc52ac9 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py +++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -14,6 +14,7 @@ from typing import Any import torch import torch.nn as nn import torch.nn.functional as F +from torch.nn.attention import SDPBackend, sdpa_kernel import sglang.multimodal_gen.envs as envs from sglang.multimodal_gen.runtime.platforms.interface import ( @@ -23,6 +24,7 @@ from sglang.multimodal_gen.runtime.platforms.interface import ( PlatformEnum, ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.srt.utils import is_gfx1250_supported logger = init_logger(__name__) @@ -244,6 +246,15 @@ class RocmPlatform(Platform): exc_info=True, ) + if is_gfx1250_supported(): + count = cls._force_math_sdpa_in_attention(vae) + if count > 0: + logger.info( + "Pinned %d VAE attention modules to the math SDPA backend " + "(AOTriton fails above head_dim 256 on gfx1250)", + count, + ) + use_bf16 = envs.SGLANG_USE_ROCM_VAE_CONV2D_BF16 use_conv2d = envs.SGLANG_USE_ROCM_VAE_CONV2D or use_bf16 if use_conv2d: @@ -259,6 +270,35 @@ class RocmPlatform(Platform): return vae + @staticmethod + def _force_math_sdpa_in_attention(module: torch.nn.Module) -> int: + """Wrap VAE attention forwards so SDPA resolves to the math backend. + + On gfx1250, AOTriton's flash and mem_efficient SDPA backends fail when + head_dim > 256. On ROCm 10.0 (AOTriton 0.13.50) the VAE returns wrong + values with no error. On ROCm 10.1 (0.14.50) it raises + hipErrorProfilerNotInitialized after the full denoise. + """ + count = 0 + for child in module.modules(): + if getattr(child, "_sgl_math_sdpa", False): + continue + has_qkv = hasattr(child, "to_qkv") or all( + getattr(child, a, None) is not None for a in ("to_q", "to_k", "to_v") + ) + if not has_qkv: + continue + original = child.forward + + def wrapped(*args, _orig=original, **kwargs): + with sdpa_kernel(SDPBackend.MATH): + return _orig(*args, **kwargs) + + child.forward = wrapped + child._sgl_math_sdpa = True + count += 1 + return count + @staticmethod def _replace_groupnorm(module: torch.nn.Module, aiter_gn_cls: type) -> int: count = 0