From bbc77e7db49c6bb9b36491ab6b51e7407776c4bf Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 09:27:39 +0800 Subject: [PATCH] [Perf] diffusion: Cut LTX-2.3 per-block elementwise work (contiguous AdaLN values, gate-folded masks, vectorized head gate) (#46110) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../sglang/kernels/ops/diffusion/__init__.py | 8 ++ .../diffusion/modulate/head_gate_triton.py | 74 +++++++++++ .../runtime/models/dits/ltx_2.py | 117 ++++++++---------- .../test/unit/test_ltx2_ada_values_layout.py | 47 +++++++ .../kernels/ops/diffusion/test_head_gate.py | 48 +++++++ 5 files changed, 229 insertions(+), 65 deletions(-) create mode 100644 python/sglang/kernels/ops/diffusion/modulate/head_gate_triton.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_ltx2_ada_values_layout.py create mode 100644 test/registered/kernels/ops/diffusion/test_head_gate.py diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index a66265abab..febd91fc1f 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -279,6 +279,12 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = ( "modulate.ltx2_ada_values_triton:ltx2_ada_values9", _CUDA, ), + ( + "diffusion.head_gate_mul", + KernelBackend.TRITON, + "modulate.head_gate_triton:head_gate_mul", + _CUDA, + ), ( "diffusion.fused_inplace_qknorm_rope", KernelBackend.JIT, @@ -714,6 +720,8 @@ _EXPORTS: dict[str, str] = { "indexed_gate_fp32_": "modulate.indexed_modulation_triton", "indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton", "ltx2_ada_values9": "modulate.ltx2_ada_values_triton", + "can_use_head_gate_mul": "modulate.head_gate_triton", + "head_gate_mul": "modulate.head_gate_triton", "can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit", "modulate_scale_shift": "modulate.modulate_scale_shift_jit", "modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit", diff --git a/python/sglang/kernels/ops/diffusion/modulate/head_gate_triton.py b/python/sglang/kernels/ops/diffusion/modulate/head_gate_triton.py new file mode 100644 index 0000000000..cfb3981ab4 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/modulate/head_gate_triton.py @@ -0,0 +1,74 @@ +"""Per-head attention gate ``out * (2 * sigmoid(gate_logits))`` for LTX-2.3. + +Eager broadcasts a ``[B, T, H, 1]`` gate over ``[B, T, H, D]``, which PyTorch +runs as a non-vectorized strided kernel, plus a separate ``2 *`` pass. This +kernel takes the eager ``sigmoid(gate_logits)`` and applies the doubling and +the product in fp32 with one bf16 rounding, the same operations and rounding +as eager, so the output is bit-identical. +""" + +import math + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _head_gate_mul_kernel( + out_ptr, + x_ptr, + gate_ptr, + hidden: tl.constexpr, + num_heads: tl.constexpr, + head_dim: tl.constexpr, + BLOCK_N: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, BLOCK_N) + mask = cols < hidden + x = tl.load(x_ptr + row * hidden + cols, mask=mask, other=0.0).to(tl.float32) + gate = tl.load( + gate_ptr + row * num_heads + cols // head_dim, mask=mask, other=0.0 + ).to(tl.float32) + # eager rounds 2 * sigmoid to bf16 first; doubling a bf16 value is exact + y = x * (gate * 2.0) + tl.store(out_ptr + row * hidden + cols, y.to(tl.bfloat16), mask=mask) + + +def can_use_head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> bool: + """``x``: contiguous ``[B, T, H, D]`` or ``[B, T, H * D]``; ``gate``: ``[B, T, H]``.""" + return ( + x.is_cuda + and x.dtype == torch.bfloat16 + and gate.dtype == torch.bfloat16 + and x.device == gate.device + and x.is_contiguous() + and gate.is_contiguous() + and x.dim() in (3, 4) + and gate.dim() == 3 + and x.shape[:2] == gate.shape[:2] + and math.prod(x.shape[2:]) % gate.shape[-1] == 0 + and math.prod(x.shape[2:]) <= 16384 + ) + + +def head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + """``x * (2 * gate)`` per head, returned as ``[B, T, H * D]``.""" + x = x.view(x.shape[0], x.shape[1], -1) + hidden = int(x.shape[-1]) + num_heads = int(gate.shape[-1]) + out = torch.empty_like(x) + rows = int(x.shape[0] * x.shape[1]) + if rows: + _head_gate_mul_kernel[(rows,)]( + out, + x, + gate, + hidden, + num_heads, + hidden // num_heads, + BLOCK_N=triton.next_power_of_2(hidden), + num_warps=4 if hidden <= 4096 else 8, + ) + return out diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index f5d5b4c216..23db3edc95 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -16,11 +16,13 @@ from sglang.kernels.ops.diffusion import ( LTX2_RMS_NORM_MODULATE, BitExactFusionGate, can_use_fused_rmsnorm_modulation, + can_use_head_gate_mul, can_use_linear_gelu, can_use_ltx2_qknorm_split_rope_cuda, can_use_modulate_scale_shift_cuda, fused_linear_gelu_tanh, fused_ltx2_rms_norm_modulate, + head_gate_mul, ltx2_qknorm_split_rope_cuda, modulate_scale_shift_cuda, residual_gate_add, @@ -1009,10 +1011,14 @@ class LTX2Attention(nn.Module): if self.to_gate_logits is not None: gate_logits, _ = self.to_gate_logits(gate_input) + gate = torch.sigmoid(gate_logits) b, t = out.shape[:2] - out = out.view(b, t, self.local_heads, self.dim_head) - out = out * (2.0 * torch.sigmoid(gate_logits).unsqueeze(-1)) - out = out.view(b, t, self.local_heads * self.dim_head) + if can_use_head_gate_mul(out, gate): + out = head_gate_mul(out, gate) + else: + out = out.view(b, t, self.local_heads, self.dim_head) + out = out * (2.0 * gate.unsqueeze(-1)) + out = out.view(b, t, self.local_heads * self.dim_head) out_flat = out.flatten(2) out_proj, _ = self.to_out[0](out_flat) @@ -1250,16 +1256,21 @@ class LTX2TransformerBlock(nn.Module): indices: slice, ) -> tuple[torch.Tensor, ...]: num_ada_params = int(scale_shift_table.shape[0]) - ada_values = ( - scale_shift_table[indices] - .unsqueeze(0) - .unsqueeze(0) - .to(device=timestep.device, dtype=timestep.dtype) - + timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[ - :, :, indices, : - ] - ).unbind(dim=2) - return [t.squeeze(2) for t in ada_values] + table = scale_shift_table[indices].to( + device=timestep.device, dtype=timestep.dtype + ) + temb = timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[ + :, :, indices, : + ] + # value-major, so every returned [B, T, D] is contiguous and the modulate + # kernels take it without a copy (a [B, T, k, D] slice is strided for B > 1) + ada_values = torch.empty( + (table.shape[0], *temb.shape[:2], temb.shape[-1]), + dtype=temb.dtype, + device=temb.device, + ) + torch.add(table[:, None, None, :], temb.permute(2, 0, 1, 3), out=ada_values) + return list(ada_values.unbind(0)) def forward( self, @@ -1433,62 +1444,40 @@ class LTX2TransformerBlock(nn.Module): norm_audio_hidden_states = self.rms_norm(audio_hidden_states, self.norm_eps) # Compute combined ada params - video_per_layer_ca_scale_shift = self.video_a2v_cross_attn_scale_shift_table[ - :4, : - ] - video_per_layer_ca_gate = self.video_a2v_cross_attn_scale_shift_table[4:, :] - - video_ca_scale_shift_table = ( - video_per_layer_ca_scale_shift[None, None, :, :].to( - dtype=temb_ca_scale_shift.dtype, device=temb_ca_scale_shift.device - ) - + temb_ca_scale_shift.reshape( - batch_size, temb_ca_scale_shift.shape[1], 4, -1 - ) - ).unbind(dim=2) - video_ca_gate = ( - video_per_layer_ca_gate[None, None, :, :].to( - dtype=temb_ca_gate.dtype, device=temb_ca_gate.device - ) - + temb_ca_gate.reshape(batch_size, temb_ca_gate.shape[1], 1, -1) - ).unbind(dim=2) - ( video_a2v_ca_scale, video_a2v_ca_shift, video_v2a_ca_scale, video_v2a_ca_shift, - ) = [t.squeeze(2) for t in video_ca_scale_shift_table] - a2v_gate = video_ca_gate[0].squeeze(2) - - audio_per_layer_ca_scale_shift = self.audio_a2v_cross_attn_scale_shift_table[ - :4, : - ] - audio_per_layer_ca_gate = self.audio_a2v_cross_attn_scale_shift_table[4:, :] - - audio_ca_scale_shift_table = ( - audio_per_layer_ca_scale_shift[None, None, :, :].to( - dtype=temb_ca_audio_scale_shift.dtype, - device=temb_ca_audio_scale_shift.device, - ) - + temb_ca_audio_scale_shift.reshape( - batch_size, temb_ca_audio_scale_shift.shape[1], 4, -1 - ) - ).unbind(dim=2) - audio_ca_gate = ( - audio_per_layer_ca_gate[None, None, :, :].to( - dtype=temb_ca_audio_gate.dtype, device=temb_ca_audio_gate.device - ) - + temb_ca_audio_gate.reshape(batch_size, temb_ca_audio_gate.shape[1], 1, -1) - ).unbind(dim=2) - + ) = self.get_ada_values( + self.video_a2v_cross_attn_scale_shift_table[:4], + batch_size, + temb_ca_scale_shift, + slice(None), + ) + (a2v_gate,) = self.get_ada_values( + self.video_a2v_cross_attn_scale_shift_table[4:], + batch_size, + temb_ca_gate, + slice(None), + ) ( audio_a2v_ca_scale, audio_a2v_ca_shift, audio_v2a_ca_scale, audio_v2a_ca_shift, - ) = [t.squeeze(2) for t in audio_ca_scale_shift_table] - v2a_gate = audio_ca_gate[0].squeeze(2) + ) = self.get_ada_values( + self.audio_a2v_cross_attn_scale_shift_table[:4], + batch_size, + temb_ca_audio_scale_shift, + slice(None), + ) + (v2a_gate,) = self.get_ada_values( + self.audio_a2v_cross_attn_scale_shift_table[4:], + batch_size, + temb_ca_audio_gate, + slice(None), + ) # A2V mod_norm_hidden_states = _ltx2_modulate( @@ -1508,9 +1497,9 @@ class LTX2TransformerBlock(nn.Module): skip_sequence_parallel_override=audio_replicated_for_sp, ) if a2v_cross_attn_perturbation_mask is not None: - a2v_attn_hidden_states = ( - a2v_attn_hidden_states * a2v_cross_attn_perturbation_mask - ) + # the 0/1 mask commutes with the gate, so scale the [B, T, D] gate + # instead of the attention output + a2v_gate = a2v_gate * a2v_cross_attn_perturbation_mask hidden_states = residual_gate_add( hidden_states, a2v_attn_hidden_states, a2v_gate ) @@ -1534,9 +1523,7 @@ class LTX2TransformerBlock(nn.Module): context_replicated_prefix_len=video_memory_prefix_len, ) if v2a_cross_attn_perturbation_mask is not None: - v2a_attn_hidden_states = ( - v2a_attn_hidden_states * v2a_cross_attn_perturbation_mask - ) + v2a_gate = v2a_gate * v2a_cross_attn_perturbation_mask audio_hidden_states = residual_gate_add( audio_hidden_states, v2a_attn_hidden_states, v2a_gate ) diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_ada_values_layout.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_ada_values_layout.py new file mode 100644 index 0000000000..122c613ffc --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_ada_values_layout.py @@ -0,0 +1,47 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.multimodal_gen.runtime.models.dits.ltx_2 import LTX2TransformerBlock + + +def _reference_ada_values(table, batch_size, timestep, indices): + num = int(table.shape[0]) + values = ( + table[indices].unsqueeze(0).unsqueeze(0).to(dtype=timestep.dtype) + + timestep.reshape(batch_size, timestep.shape[1], num, -1)[:, :, indices, :] + ).unbind(dim=2) + return [t.squeeze(2) for t in values] + + +class TestLTX2AdaValuesLayout(unittest.TestCase): + def test_values_are_contiguous_and_bitwise_equal(self): + torch.manual_seed(0) + for table_dtype, tokens, indices in ( + (torch.float32, 1, slice(0, 3)), + (torch.bfloat16, 1, slice(None)), + (torch.float32, 5, slice(6, 9)), + ): + table = torch.randn(9, 16, dtype=table_dtype) + timestep = torch.randn(2, tokens, 9 * 16, dtype=torch.bfloat16) + actual = LTX2TransformerBlock.get_ada_values( + SimpleNamespace(), table, 2, timestep, indices + ) + expected = _reference_ada_values(table, 2, timestep, indices) + self.assertEqual(len(actual), len(expected)) + for a, e in zip(actual, expected): + self.assertTrue(a.is_contiguous()) + self.assertTrue(torch.equal(a, e)) + + def test_zero_one_mask_commutes_with_the_gate(self): + torch.manual_seed(0) + x = torch.randn(2, 6, 16, dtype=torch.bfloat16) + attn = torch.randn(2, 6, 16, dtype=torch.bfloat16) + gate = torch.randn(2, 1, 16, dtype=torch.bfloat16) + mask = torch.tensor([1.0, 0.0], dtype=torch.bfloat16).view(2, 1, 1) + self.assertTrue(torch.equal(x + (attn * mask) * gate, x + attn * (gate * mask))) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/diffusion/test_head_gate.py b/test/registered/kernels/ops/diffusion/test_head_gate.py new file mode 100644 index 0000000000..d41820be17 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_head_gate.py @@ -0,0 +1,48 @@ +"""``diffusion.head_gate_mul``: LTX-2.3's per-head attention gate, bit-exact with eager.""" + +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import can_use_head_gate_mul, head_gate_mul +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@pytest.mark.parametrize( + "batch,tokens,heads,head_dim", + [(2, 1536, 32, 128), (2, 126, 32, 64), (1, 7, 4, 16), (3, 5, 8, 24)], +) +@pytest.mark.parametrize("flat", [False, True]) +def test_head_gate_mul_matches_eager_bitwise( + batch: int, tokens: int, heads: int, head_dim: int, flat: bool +) -> None: + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + torch.manual_seed(0) + out = torch.randn(batch, tokens, heads, head_dim, device="cuda") * 3 + out = out.to(torch.bfloat16) + logits = (torch.randn(batch, tokens, heads, device="cuda") * 4).to(torch.bfloat16) + gate = torch.sigmoid(logits) + expected = (out * (2.0 * gate.unsqueeze(-1))).view(batch, tokens, -1) + x = out.view(batch, tokens, -1) if flat else out + assert can_use_head_gate_mul(x, gate) + actual = head_gate_mul(x, gate) + assert actual.shape == expected.shape + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + + +def test_head_gate_mul_rejects_strided_or_mismatched_inputs() -> None: + if not torch.cuda.is_available(): + pytest.skip("CUDA required") + out = torch.randn(2, 6, 4, 8, device="cuda", dtype=torch.bfloat16) + gate = torch.rand(2, 6, 4, device="cuda", dtype=torch.bfloat16) + assert not can_use_head_gate_mul(out.transpose(1, 2), gate) + assert not can_use_head_gate_mul(out, gate[:, :5]) + assert not can_use_head_gate_mul(out.float(), gate) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))