[Perf] [XPU] diffusion: Use the AOT SYCL fused residual-norm-scale-shift kernel (#41401)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
904691e0b5
commit
76c382e80f
@@ -636,8 +636,10 @@ _EXPORTS: dict[str, str] = {
|
||||
"can_use_flux2_gated_resnorm": "norm.flux2_gated_resnorm_jit",
|
||||
"flux2_gated_resnorm_raw": "norm.flux2_gated_resnorm_jit",
|
||||
"FLYDSL_NORM_MIN_ALIGNED_DIM": "norm.fused_residual_norm_flydsl",
|
||||
"can_use_fused_scale_residual_norm_scale_shift_sycl": "norm.scale_residual_norm_scale_shift_sycl",
|
||||
"can_use_fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
|
||||
"flydsl_fused_residual_norm_scale_shift": "norm.fused_residual_norm_flydsl",
|
||||
"fused_scale_residual_norm_scale_shift_sycl": "norm.scale_residual_norm_scale_shift_sycl",
|
||||
"fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
|
||||
"flydsl_norm_scale_shift": "norm.fused_residual_norm_flydsl",
|
||||
"apply_group_norm_silu": "norm.group_norm_silu",
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import sgl_kernel # noqa: F401
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.diffusion.norm.scale_residual_norm_scale_shift_triton import (
|
||||
can_use_fused_scale_residual_norm_scale_shift_triton,
|
||||
)
|
||||
from sglang.srt.utils.patch_torch import register_fake_if_exists
|
||||
|
||||
_fused_scale_residual_norm_scale_shift = (
|
||||
torch.ops.sgl_kernel.fused_scale_residual_norm_scale_shift
|
||||
)
|
||||
|
||||
|
||||
def _param_dtype(
|
||||
*, gate: torch.Tensor | int, weight: torch.Tensor | None, shift: torch.Tensor
|
||||
) -> torch.dtype:
|
||||
if weight is not None:
|
||||
return weight.dtype
|
||||
if isinstance(gate, torch.Tensor):
|
||||
return gate.dtype
|
||||
return shift.dtype
|
||||
|
||||
|
||||
def can_use_fused_scale_residual_norm_scale_shift_sycl(
|
||||
*,
|
||||
residual: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor | int,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
weight: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
) -> bool:
|
||||
if not can_use_fused_scale_residual_norm_scale_shift_triton(
|
||||
residual=residual,
|
||||
x=x,
|
||||
gate=gate,
|
||||
shift=shift,
|
||||
scale=scale,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
):
|
||||
return False
|
||||
dtypes = {p.dtype for p in (gate, weight, bias) if isinstance(p, torch.Tensor)}
|
||||
if len(dtypes) > 1:
|
||||
return False
|
||||
param_dtype = _param_dtype(gate=gate, weight=weight, shift=shift)
|
||||
return all(
|
||||
m.dtype == param_dtype or param_dtype == torch.float32 for m in (shift, scale)
|
||||
)
|
||||
|
||||
|
||||
def fused_scale_residual_norm_scale_shift_sycl(
|
||||
*,
|
||||
residual: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor | int,
|
||||
shift: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
weight: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
param_dtype = _param_dtype(gate=gate, weight=weight, shift=shift)
|
||||
return _fused_scale_residual_norm_scale_shift(
|
||||
residual,
|
||||
x,
|
||||
gate if isinstance(gate, torch.Tensor) else None,
|
||||
weight,
|
||||
bias,
|
||||
scale.to(param_dtype),
|
||||
shift.to(param_dtype),
|
||||
eps,
|
||||
)
|
||||
|
||||
|
||||
@register_fake_if_exists("sgl_kernel::fused_scale_residual_norm_scale_shift")
|
||||
def _(
|
||||
residual: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
gate: torch.Tensor | None,
|
||||
weight: torch.Tensor | None,
|
||||
bias: torch.Tensor | None,
|
||||
scale: torch.Tensor,
|
||||
shift: torch.Tensor,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(x), torch.empty_like(x)
|
||||
@@ -90,6 +90,11 @@ if USE_AITER:
|
||||
if _is_xpu:
|
||||
from sgl_kernel import fused_inplace_qknorm_rope
|
||||
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
can_use_fused_scale_residual_norm_scale_shift_sycl,
|
||||
fused_scale_residual_norm_scale_shift_sycl,
|
||||
)
|
||||
|
||||
if not _is_cpu:
|
||||
from sglang.kernels.ops.diffusion import norm_infer, rms_norm_fn
|
||||
|
||||
@@ -727,6 +732,25 @@ class _ScaleResidualNormScaleShift(CustomOp):
|
||||
if self.norm_type == "layer":
|
||||
weight = self.norm.weight
|
||||
bias = self.norm.bias
|
||||
if can_use_fused_scale_residual_norm_scale_shift_sycl(
|
||||
residual=residual,
|
||||
x=x,
|
||||
gate=gate,
|
||||
shift=shift,
|
||||
scale=scale,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
):
|
||||
return fused_scale_residual_norm_scale_shift_sycl(
|
||||
residual=residual,
|
||||
x=x,
|
||||
gate=gate,
|
||||
shift=shift,
|
||||
scale=scale,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
eps=self.eps,
|
||||
)
|
||||
if can_use_fused_scale_residual_norm_scale_shift_triton(
|
||||
residual=residual,
|
||||
x=x,
|
||||
|
||||
Reference in New Issue
Block a user