[Diffusion] Fuse Anima AdaLN modulation and cache the FFI callable (#42608)
This commit is contained in:
@@ -15,7 +15,7 @@ from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
_SUPPORTED_DTYPES = (torch.float16, torch.bfloat16)
|
||||
@@ -23,10 +23,11 @@ _ALIGN_BYTES = 16
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module:
|
||||
def _jit_modulate_scale_shift_kernel(dtype: torch.dtype) -> Callable:
|
||||
if dtype not in _SUPPORTED_DTYPES:
|
||||
raise RuntimeError(f"Unsupported modulate_scale_shift dtype: {dtype}")
|
||||
args = make_cpp_args(dtype)
|
||||
# Cache the exported function too, avoiding repeated FFI attribute lookup.
|
||||
return load_jit(
|
||||
"diffusion_modulate_scale_shift",
|
||||
*args,
|
||||
@@ -37,7 +38,7 @@ def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module:
|
||||
f"modulate_scale_shift::ModulateScaleShiftKernel<{args}>::run",
|
||||
),
|
||||
],
|
||||
)
|
||||
).modulate_scale_shift
|
||||
|
||||
|
||||
def _fake_impl(
|
||||
@@ -56,8 +57,8 @@ def modulate_scale_shift_cuda(
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA."""
|
||||
out = torch.empty_like(x)
|
||||
module = _jit_modulate_scale_shift_module(x.dtype)
|
||||
module.modulate_scale_shift(out, x, scale, shift)
|
||||
kernel = _jit_modulate_scale_shift_kernel(x.dtype)
|
||||
kernel(out, x, scale, shift)
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,11 @@ import torch
|
||||
from diffusers.models.embeddings import Timesteps
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.diffusion import BitExactFusionGate, tensors_equal
|
||||
from sglang.kernels.ops.diffusion import (
|
||||
BitExactFusionGate,
|
||||
modulate_scale_shift,
|
||||
tensors_equal,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
|
||||
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
|
||||
@@ -164,7 +168,23 @@ class AnimaAdaLayerNorm(nn.Module):
|
||||
modulation = self.linear_2(modulation)[0]
|
||||
modulation = modulation + temb[..., : modulation.shape[-1]]
|
||||
values = modulation.unsqueeze(1).chunk(3 if self.gated else 2, dim=-1)
|
||||
out = self.norm(x) * (1 + values[1]) + values[0]
|
||||
norm_x = self.norm(x)
|
||||
if (
|
||||
norm_x.is_cuda
|
||||
and norm_x.dtype in (torch.float16, torch.bfloat16)
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
# Batched chunk views are strided; only copy the small [B, D]
|
||||
# modulation rows, never the [B, S, D] activation.
|
||||
out = modulate_scale_shift(
|
||||
norm_x,
|
||||
values[1].squeeze(1).contiguous(),
|
||||
values[0].squeeze(1).contiguous(),
|
||||
)
|
||||
else:
|
||||
# Keep the original graph for torch.compile: the CUDA dispatch
|
||||
# checks pointer alignment, which cannot be traced in a full graph.
|
||||
out = norm_x * (1 + values[1]) + values[0]
|
||||
return (out, values[2]) if self.gated else out
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user