[AMD][diffusion] VAE attention to the math SDPA backend on gfx1250 (#42858)
This commit is contained in:
@@ -14,6 +14,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
|
||||
import sglang.multimodal_gen.envs as envs
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import (
|
||||
@@ -23,6 +24,7 @@ from sglang.multimodal_gen.runtime.platforms.interface import (
|
||||
PlatformEnum,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.utils import is_gfx1250_supported
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -244,6 +246,15 @@ class RocmPlatform(Platform):
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
if is_gfx1250_supported():
|
||||
count = cls._force_math_sdpa_in_attention(vae)
|
||||
if count > 0:
|
||||
logger.info(
|
||||
"Pinned %d VAE attention modules to the math SDPA backend "
|
||||
"(AOTriton fails above head_dim 256 on gfx1250)",
|
||||
count,
|
||||
)
|
||||
|
||||
use_bf16 = envs.SGLANG_USE_ROCM_VAE_CONV2D_BF16
|
||||
use_conv2d = envs.SGLANG_USE_ROCM_VAE_CONV2D or use_bf16
|
||||
if use_conv2d:
|
||||
@@ -259,6 +270,35 @@ class RocmPlatform(Platform):
|
||||
|
||||
return vae
|
||||
|
||||
@staticmethod
|
||||
def _force_math_sdpa_in_attention(module: torch.nn.Module) -> int:
|
||||
"""Wrap VAE attention forwards so SDPA resolves to the math backend.
|
||||
|
||||
On gfx1250, AOTriton's flash and mem_efficient SDPA backends fail when
|
||||
head_dim > 256. On ROCm 10.0 (AOTriton 0.13.50) the VAE returns wrong
|
||||
values with no error. On ROCm 10.1 (0.14.50) it raises
|
||||
hipErrorProfilerNotInitialized after the full denoise.
|
||||
"""
|
||||
count = 0
|
||||
for child in module.modules():
|
||||
if getattr(child, "_sgl_math_sdpa", False):
|
||||
continue
|
||||
has_qkv = hasattr(child, "to_qkv") or all(
|
||||
getattr(child, a, None) is not None for a in ("to_q", "to_k", "to_v")
|
||||
)
|
||||
if not has_qkv:
|
||||
continue
|
||||
original = child.forward
|
||||
|
||||
def wrapped(*args, _orig=original, **kwargs):
|
||||
with sdpa_kernel(SDPBackend.MATH):
|
||||
return _orig(*args, **kwargs)
|
||||
|
||||
child.forward = wrapped
|
||||
child._sgl_math_sdpa = True
|
||||
count += 1
|
||||
return count
|
||||
|
||||
@staticmethod
|
||||
def _replace_groupnorm(module: torch.nn.Module, aiter_gn_cls: type) -> int:
|
||||
count = 0
|
||||
|
||||
Reference in New Issue
Block a user