[AMD][Diffusion] Route unquantized linears through aiter tgemm on gfx1250 (#41988)

This commit is contained in:
sogalin_codegen
2026-10-15 21:08:44 +08:00
committed by GitHub
parent abf92826ee
commit 2092160d42
2 changed files with 20 additions and 0 deletions
+4
View File
@@ -135,6 +135,7 @@ if TYPE_CHECKING:
SGLANG_USE_ROCM_CUDNN_BENCHMARK: bool = False
SGLANG_USE_ROCM_VAE_CONV2D: bool = False
SGLANG_USE_ROCM_VAE_CONV2D_BF16: bool = False
SGLANG_USE_ROCM_AITER_TGEMM: bool = False
def get_default_cache_root() -> str:
@@ -536,6 +537,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
"SGLANG_USE_ROCM_VAE_CONV2D": _lazy_bool("SGLANG_USE_ROCM_VAE_CONV2D"),
# ROCm: use BF16 compute for the Conv2D replacement (implies CONV2D=true)
"SGLANG_USE_ROCM_VAE_CONV2D_BF16": _lazy_bool("SGLANG_USE_ROCM_VAE_CONV2D_BF16"),
# ROCm: route unquantized bf16/fp16 linears through aiter tgemm, which picks
# the backend per (M, N, K) from aiter's bf16_tuned_gemm CSVs
"SGLANG_USE_ROCM_AITER_TGEMM": _lazy_bool("SGLANG_USE_ROCM_AITER_TGEMM"),
}
# Add cache-dit Secondary Transformer Env Vars via programmatic generation to reduce duplication
@@ -11,6 +11,7 @@ import torch.nn.functional as F
from torch.nn.parameter import Parameter
from sglang.kernels.kernel_api_logging import wrap_method_with_debug_kernel_once
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed import (
divide,
get_tp_group,
@@ -42,6 +43,11 @@ from sglang.multimodal_gen.runtime.utils.weight_attrs import set_weight_attrs
logger = init_logger(__name__)
IS_AMP_SUPPORTED = current_platform.is_amp_supported()
_USE_AITER_TGEMM = current_platform.is_rocm() and envs.SGLANG_USE_ROCM_AITER_TGEMM
if _USE_AITER_TGEMM:
from aiter.tuned_gemm import tgemm
# aiter gemm_a16w16 kernels index operands with int32 offsets and fault beyond this.
_AITER_TGEMM_MAX_NUMEL = 2**31
WEIGHT_LOADER_V2_SUPPORTED = [
"CompressedTensorsLinearMethod",
"AWQMarlinLinearMethod",
@@ -165,6 +171,16 @@ def apply_unquantized_linear(
None if bias is None else bias.to(torch.float32),
).to(x.dtype)
if (
_USE_AITER_TGEMM
and x.dtype in (torch.bfloat16, torch.float16)
and weight.dtype == x.dtype
and (bias is None or bias.dtype == x.dtype)
and x.numel() < _AITER_TGEMM_MAX_NUMEL
and x.numel() // x.shape[-1] * weight.shape[0] < _AITER_TGEMM_MAX_NUMEL
):
return tgemm.mm(x, weight, bias)
return (
F.linear(x, weight, bias)
if IS_AMP_SUPPORTED or bias is None