diff --git a/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py b/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py index 1f13f7bf07..2a5617b897 100644 --- a/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py +++ b/python/sglang/kernels/ops/diffusion/modulate/modulate_scale_shift_jit.py @@ -15,7 +15,7 @@ from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: - from tvm_ffi.module import Module + from collections.abc import Callable _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) @@ -23,10 +23,11 @@ _ALIGN_BYTES = 16 @cache_once -def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module: +def _jit_modulate_scale_shift_kernel(dtype: torch.dtype) -> Callable: if dtype not in _SUPPORTED_DTYPES: raise RuntimeError(f"Unsupported modulate_scale_shift dtype: {dtype}") args = make_cpp_args(dtype) + # Cache the exported function too, avoiding repeated FFI attribute lookup. return load_jit( "diffusion_modulate_scale_shift", *args, @@ -37,7 +38,7 @@ def _jit_modulate_scale_shift_module(dtype: torch.dtype) -> Module: f"modulate_scale_shift::ModulateScaleShiftKernel<{args}>::run", ), ], - ) + ).modulate_scale_shift def _fake_impl( @@ -56,8 +57,8 @@ def modulate_scale_shift_cuda( ) -> torch.Tensor: """Fused ``x * (1 + scale[:, None]) + shift[:, None]``; validated by CUDA.""" out = torch.empty_like(x) - module = _jit_modulate_scale_shift_module(x.dtype) - module.modulate_scale_shift(out, x, scale, shift) + kernel = _jit_modulate_scale_shift_kernel(x.dtype) + kernel(out, x, scale, shift) return out diff --git a/python/sglang/multimodal_gen/runtime/models/dits/anima.py b/python/sglang/multimodal_gen/runtime/models/dits/anima.py index 18e9ef3064..a58857c4b9 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/anima.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/anima.py @@ -8,7 +8,11 @@ import torch from diffusers.models.embeddings import Timesteps from torch import nn -from sglang.kernels.ops.diffusion import BitExactFusionGate, tensors_equal +from sglang.kernels.ops.diffusion import ( + BitExactFusionGate, + modulate_scale_shift, + tensors_equal, +) from sglang.multimodal_gen.configs.models.dits.anima import AnimaDiTConfig from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in from sglang.multimodal_gen.runtime.distributed import get_tp_world_size @@ -164,7 +168,23 @@ class AnimaAdaLayerNorm(nn.Module): modulation = self.linear_2(modulation)[0] modulation = modulation + temb[..., : modulation.shape[-1]] values = modulation.unsqueeze(1).chunk(3 if self.gated else 2, dim=-1) - out = self.norm(x) * (1 + values[1]) + values[0] + norm_x = self.norm(x) + if ( + norm_x.is_cuda + and norm_x.dtype in (torch.float16, torch.bfloat16) + and not torch.compiler.is_compiling() + ): + # Batched chunk views are strided; only copy the small [B, D] + # modulation rows, never the [B, S, D] activation. + out = modulate_scale_shift( + norm_x, + values[1].squeeze(1).contiguous(), + values[0].squeeze(1).contiguous(), + ) + else: + # Keep the original graph for torch.compile: the CUDA dispatch + # checks pointer alignment, which cannot be traced in a full graph. + out = norm_x * (1 + values[1]) + values[0] return (out, values[2]) if self.gated else out