diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index 5559e0bedb..79ec89c485 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -636,8 +636,10 @@ _EXPORTS: dict[str, str] = { "can_use_flux2_gated_resnorm": "norm.flux2_gated_resnorm_jit", "flux2_gated_resnorm_raw": "norm.flux2_gated_resnorm_jit", "FLYDSL_NORM_MIN_ALIGNED_DIM": "norm.fused_residual_norm_flydsl", + "can_use_fused_scale_residual_norm_scale_shift_sycl": "norm.scale_residual_norm_scale_shift_sycl", "can_use_fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton", "flydsl_fused_residual_norm_scale_shift": "norm.fused_residual_norm_flydsl", + "fused_scale_residual_norm_scale_shift_sycl": "norm.scale_residual_norm_scale_shift_sycl", "fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton", "flydsl_norm_scale_shift": "norm.fused_residual_norm_flydsl", "apply_group_norm_silu": "norm.group_norm_silu", diff --git a/python/sglang/kernels/ops/diffusion/norm/scale_residual_norm_scale_shift_sycl.py b/python/sglang/kernels/ops/diffusion/norm/scale_residual_norm_scale_shift_sycl.py new file mode 100644 index 0000000000..aaadc20853 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/norm/scale_residual_norm_scale_shift_sycl.py @@ -0,0 +1,88 @@ +import sgl_kernel # noqa: F401 +import torch + +from sglang.kernels.ops.diffusion.norm.scale_residual_norm_scale_shift_triton import ( + can_use_fused_scale_residual_norm_scale_shift_triton, +) +from sglang.srt.utils.patch_torch import register_fake_if_exists + +_fused_scale_residual_norm_scale_shift = ( + torch.ops.sgl_kernel.fused_scale_residual_norm_scale_shift +) + + +def _param_dtype( + *, gate: torch.Tensor | int, weight: torch.Tensor | None, shift: torch.Tensor +) -> torch.dtype: + if weight is not None: + return weight.dtype + if isinstance(gate, torch.Tensor): + return gate.dtype + return shift.dtype + + +def can_use_fused_scale_residual_norm_scale_shift_sycl( + *, + residual: torch.Tensor, + x: torch.Tensor, + gate: torch.Tensor | int, + shift: torch.Tensor, + scale: torch.Tensor, + weight: torch.Tensor | None, + bias: torch.Tensor | None, +) -> bool: + if not can_use_fused_scale_residual_norm_scale_shift_triton( + residual=residual, + x=x, + gate=gate, + shift=shift, + scale=scale, + weight=weight, + bias=bias, + ): + return False + dtypes = {p.dtype for p in (gate, weight, bias) if isinstance(p, torch.Tensor)} + if len(dtypes) > 1: + return False + param_dtype = _param_dtype(gate=gate, weight=weight, shift=shift) + return all( + m.dtype == param_dtype or param_dtype == torch.float32 for m in (shift, scale) + ) + + +def fused_scale_residual_norm_scale_shift_sycl( + *, + residual: torch.Tensor, + x: torch.Tensor, + gate: torch.Tensor | int, + shift: torch.Tensor, + scale: torch.Tensor, + weight: torch.Tensor | None, + bias: torch.Tensor | None, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + param_dtype = _param_dtype(gate=gate, weight=weight, shift=shift) + return _fused_scale_residual_norm_scale_shift( + residual, + x, + gate if isinstance(gate, torch.Tensor) else None, + weight, + bias, + scale.to(param_dtype), + shift.to(param_dtype), + eps, + ) + + +@register_fake_if_exists("sgl_kernel::fused_scale_residual_norm_scale_shift") +def _( + residual: torch.Tensor, + x: torch.Tensor, + gate: torch.Tensor | None, + weight: torch.Tensor | None, + bias: torch.Tensor | None, + scale: torch.Tensor, + shift: torch.Tensor, + eps: float, +) -> tuple[torch.Tensor, torch.Tensor]: + return torch.empty_like(x), torch.empty_like(x) diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index b9c9ed372d..dfb0f12004 100755 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -90,6 +90,11 @@ if USE_AITER: if _is_xpu: from sgl_kernel import fused_inplace_qknorm_rope + from sglang.kernels.ops.diffusion import ( + can_use_fused_scale_residual_norm_scale_shift_sycl, + fused_scale_residual_norm_scale_shift_sycl, + ) + if not _is_cpu: from sglang.kernels.ops.diffusion import norm_infer, rms_norm_fn @@ -727,6 +732,25 @@ class _ScaleResidualNormScaleShift(CustomOp): if self.norm_type == "layer": weight = self.norm.weight bias = self.norm.bias + if can_use_fused_scale_residual_norm_scale_shift_sycl( + residual=residual, + x=x, + gate=gate, + shift=shift, + scale=scale, + weight=weight, + bias=bias, + ): + return fused_scale_residual_norm_scale_shift_sycl( + residual=residual, + x=x, + gate=gate, + shift=shift, + scale=scale, + weight=weight, + bias=bias, + eps=self.eps, + ) if can_use_fused_scale_residual_norm_scale_shift_triton( residual=residual, x=x,