[Diffusion] Fuse Anima AdaLN modulation and cache the FFI callable (#42608)

This commit is contained in:
CuzMi
2026-10-07 12:01:59 +08:00
committed by GitHub
parent c0df52be2d
commit bde01f8dc8
2 changed files with 28 additions and 7 deletions
@@ -15,7 +15,7 @@ from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.srt.utils.custom_op import register_custom_op
if TYPE_CHECKING:
from tvm_ffi.module import Module
from collections.abc import Callable
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
@@ -23,10 +23,11 @@ _ALIGN_BYTES = 16
@cache_once
def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module:
def _jit_modulate_scale_shift_kernel(dtype: torch.dtype) -> Callable:
if dtype not in _SUPPORTED_DTYPES:
raise RuntimeError(f"Unsupported modulate_scale_shift dtype: {dtype}")
args = make_cpp_args(dtype)
# Cache the exported function too, avoiding repeated FFI attribute lookup.
return load_jit(
"diffusion_modulate_scale_shift",
*args,
@@ -37,7 +38,7 @@ def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module:
f"modulate_scale_shift::ModulateScaleShiftKernel<{args}>::run",
),
],
)
).modulate_scale_shift
def _fake_impl(
@@ -56,8 +57,8 @@ def modulate_scale_shift_cuda(
) -> torch.Tensor:
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA."""
out = torch.empty_like(x)
module = _jit_modulate_scale_shift_module(x.dtype)
module.modulate_scale_shift(out, x, scale, shift)
kernel = _jit_modulate_scale_shift_kernel(x.dtype)
kernel(out, x, scale, shift)
return out
@@ -8,7 +8,11 @@ import torch
from diffusers.models.embeddings import Timesteps
from torch import nn
from sglang.kernels.ops.diffusion import BitExactFusionGate, tensors_equal
from sglang.kernels.ops.diffusion import (
BitExactFusionGate,
modulate_scale_shift,
tensors_equal,
)
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
@@ -164,7 +168,23 @@ class AnimaAdaLayerNorm(nn.Module):
modulation = self.linear_2(modulation)[0]
modulation = modulation + temb[..., : modulation.shape[-1]]
values = modulation.unsqueeze(1).chunk(3 if self.gated else 2, dim=-1)
out = self.norm(x) * (1 + values[1]) + values[0]
norm_x = self.norm(x)
if (
norm_x.is_cuda
and norm_x.dtype in (torch.float16, torch.bfloat16)
and not torch.compiler.is_compiling()
):
# Batched chunk views are strided; only copy the small [B, D]
# modulation rows, never the [B, S, D] activation.
out = modulate_scale_shift(
norm_x,
values[1].squeeze(1).contiguous(),
values[0].squeeze(1).contiguous(),
)
else:
# Keep the original graph for torch.compile: the CUDA dispatch
# checks pointer alignment, which cannot be traced in a full graph.
out = norm_x * (1 + values[1]) + values[0]
return (out, values[2]) if self.gated else out