[Perf] diffusion: Cut LTX-2.3 per-block elementwise work (contiguous AdaLN values, gate-folded masks, vectorized head gate) (#46110)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:27:39 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 4e2d5b17ad
commit bbc77e7db4
5 changed files with 229 additions and 65 deletions
@@ -279,6 +279,12 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
_CUDA,
),
(
"diffusion.head_gate_mul",
KernelBackend.TRITON,
"modulate.head_gate_triton:head_gate_mul",
_CUDA,
),
(
"diffusion.fused_inplace_qknorm_rope",
KernelBackend.JIT,
@@ -714,6 +720,8 @@ _EXPORTS: dict[str, str] = {
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
"can_use_head_gate_mul": "modulate.head_gate_triton",
"head_gate_mul": "modulate.head_gate_triton",
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
"modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
@@ -0,0 +1,74 @@
"""Per-head attention gate ``out * (2 * sigmoid(gate_logits))`` for LTX-2.3.
Eager broadcasts a ``[B, T, H, 1]`` gate over ``[B, T, H, D]``, which PyTorch
runs as a non-vectorized strided kernel, plus a separate ``2 *`` pass. This
kernel takes the eager ``sigmoid(gate_logits)`` and applies the doubling and
the product in fp32 with one bf16 rounding, the same operations and rounding
as eager, so the output is bit-identical.
"""
import math
import torch
import triton
import triton.language as tl
@triton.jit
def _head_gate_mul_kernel(
out_ptr,
x_ptr,
gate_ptr,
hidden: tl.constexpr,
num_heads: tl.constexpr,
head_dim: tl.constexpr,
BLOCK_N: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
cols = tl.arange(0, BLOCK_N)
mask = cols < hidden
x = tl.load(x_ptr + row * hidden + cols, mask=mask, other=0.0).to(tl.float32)
gate = tl.load(
gate_ptr + row * num_heads + cols // head_dim, mask=mask, other=0.0
).to(tl.float32)
# eager rounds 2 * sigmoid to bf16 first; doubling a bf16 value is exact
y = x * (gate * 2.0)
tl.store(out_ptr + row * hidden + cols, y.to(tl.bfloat16), mask=mask)
def can_use_head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> bool:
"""``x``: contiguous ``[B, T, H, D]`` or ``[B, T, H * D]``; ``gate``: ``[B, T, H]``."""
return (
x.is_cuda
and x.dtype == torch.bfloat16
and gate.dtype == torch.bfloat16
and x.device == gate.device
and x.is_contiguous()
and gate.is_contiguous()
and x.dim() in (3, 4)
and gate.dim() == 3
and x.shape[:2] == gate.shape[:2]
and math.prod(x.shape[2:]) % gate.shape[-1] == 0
and math.prod(x.shape[2:]) <= 16384
)
def head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
"""``x * (2 * gate)`` per head, returned as ``[B, T, H * D]``."""
x = x.view(x.shape[0], x.shape[1], -1)
hidden = int(x.shape[-1])
num_heads = int(gate.shape[-1])
out = torch.empty_like(x)
rows = int(x.shape[0] * x.shape[1])
if rows:
_head_gate_mul_kernel[(rows,)](
out,
x,
gate,
hidden,
num_heads,
hidden // num_heads,
BLOCK_N=triton.next_power_of_2(hidden),
num_warps=4 if hidden <= 4096 else 8,
)
return out
@@ -16,11 +16,13 @@ from sglang.kernels.ops.diffusion import (
LTX2_RMS_NORM_MODULATE,
BitExactFusionGate,
can_use_fused_rmsnorm_modulation,
can_use_head_gate_mul,
can_use_linear_gelu,
can_use_ltx2_qknorm_split_rope_cuda,
can_use_modulate_scale_shift_cuda,
fused_linear_gelu_tanh,
fused_ltx2_rms_norm_modulate,
head_gate_mul,
ltx2_qknorm_split_rope_cuda,
modulate_scale_shift_cuda,
residual_gate_add,
@@ -1009,10 +1011,14 @@ class LTX2Attention(nn.Module):
if self.to_gate_logits is not None:
gate_logits, _ = self.to_gate_logits(gate_input)
gate = torch.sigmoid(gate_logits)
b, t = out.shape[:2]
out = out.view(b, t, self.local_heads, self.dim_head)
out = out * (2.0 * torch.sigmoid(gate_logits).unsqueeze(-1))
out = out.view(b, t, self.local_heads * self.dim_head)
if can_use_head_gate_mul(out, gate):
out = head_gate_mul(out, gate)
else:
out = out.view(b, t, self.local_heads, self.dim_head)
out = out * (2.0 * gate.unsqueeze(-1))
out = out.view(b, t, self.local_heads * self.dim_head)
out_flat = out.flatten(2)
out_proj, _ = self.to_out[0](out_flat)
@@ -1250,16 +1256,21 @@ class LTX2TransformerBlock(nn.Module):
indices: slice,
) -> tuple[torch.Tensor, ...]:
num_ada_params = int(scale_shift_table.shape[0])
ada_values = (
scale_shift_table[indices]
.unsqueeze(0)
.unsqueeze(0)
.to(device=timestep.device, dtype=timestep.dtype)
+ timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[
:, :, indices, :
]
).unbind(dim=2)
return [t.squeeze(2) for t in ada_values]
table = scale_shift_table[indices].to(
device=timestep.device, dtype=timestep.dtype
)
temb = timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[
:, :, indices, :
]
# value-major, so every returned [B, T, D] is contiguous and the modulate
# kernels take it without a copy (a [B, T, k, D] slice is strided for B > 1)
ada_values = torch.empty(
(table.shape[0], *temb.shape[:2], temb.shape[-1]),
dtype=temb.dtype,
device=temb.device,
)
torch.add(table[:, None, None, :], temb.permute(2, 0, 1, 3), out=ada_values)
return list(ada_values.unbind(0))
def forward(
self,
@@ -1433,62 +1444,40 @@ class LTX2TransformerBlock(nn.Module):
norm_audio_hidden_states = self.rms_norm(audio_hidden_states, self.norm_eps)
# Compute combined ada params
video_per_layer_ca_scale_shift = self.video_a2v_cross_attn_scale_shift_table[
:4, :
]
video_per_layer_ca_gate = self.video_a2v_cross_attn_scale_shift_table[4:, :]
video_ca_scale_shift_table = (
video_per_layer_ca_scale_shift[None, None, :, :].to(
dtype=temb_ca_scale_shift.dtype, device=temb_ca_scale_shift.device
)
+ temb_ca_scale_shift.reshape(
batch_size, temb_ca_scale_shift.shape[1], 4, -1
)
).unbind(dim=2)
video_ca_gate = (
video_per_layer_ca_gate[None, None, :, :].to(
dtype=temb_ca_gate.dtype, device=temb_ca_gate.device
)
+ temb_ca_gate.reshape(batch_size, temb_ca_gate.shape[1], 1, -1)
).unbind(dim=2)
(
video_a2v_ca_scale,
video_a2v_ca_shift,
video_v2a_ca_scale,
video_v2a_ca_shift,
) = [t.squeeze(2) for t in video_ca_scale_shift_table]
a2v_gate = video_ca_gate[0].squeeze(2)
audio_per_layer_ca_scale_shift = self.audio_a2v_cross_attn_scale_shift_table[
:4, :
]
audio_per_layer_ca_gate = self.audio_a2v_cross_attn_scale_shift_table[4:, :]
audio_ca_scale_shift_table = (
audio_per_layer_ca_scale_shift[None, None, :, :].to(
dtype=temb_ca_audio_scale_shift.dtype,
device=temb_ca_audio_scale_shift.device,
)
+ temb_ca_audio_scale_shift.reshape(
batch_size, temb_ca_audio_scale_shift.shape[1], 4, -1
)
).unbind(dim=2)
audio_ca_gate = (
audio_per_layer_ca_gate[None, None, :, :].to(
dtype=temb_ca_audio_gate.dtype, device=temb_ca_audio_gate.device
)
+ temb_ca_audio_gate.reshape(batch_size, temb_ca_audio_gate.shape[1], 1, -1)
).unbind(dim=2)
) = self.get_ada_values(
self.video_a2v_cross_attn_scale_shift_table[:4],
batch_size,
temb_ca_scale_shift,
slice(None),
)
(a2v_gate,) = self.get_ada_values(
self.video_a2v_cross_attn_scale_shift_table[4:],
batch_size,
temb_ca_gate,
slice(None),
)
(
audio_a2v_ca_scale,
audio_a2v_ca_shift,
audio_v2a_ca_scale,
audio_v2a_ca_shift,
) = [t.squeeze(2) for t in audio_ca_scale_shift_table]
v2a_gate = audio_ca_gate[0].squeeze(2)
) = self.get_ada_values(
self.audio_a2v_cross_attn_scale_shift_table[:4],
batch_size,
temb_ca_audio_scale_shift,
slice(None),
)
(v2a_gate,) = self.get_ada_values(
self.audio_a2v_cross_attn_scale_shift_table[4:],
batch_size,
temb_ca_audio_gate,
slice(None),
)
# A2V
mod_norm_hidden_states = _ltx2_modulate(
@@ -1508,9 +1497,9 @@ class LTX2TransformerBlock(nn.Module):
skip_sequence_parallel_override=audio_replicated_for_sp,
)
if a2v_cross_attn_perturbation_mask is not None:
a2v_attn_hidden_states = (
a2v_attn_hidden_states * a2v_cross_attn_perturbation_mask
)
# the 0/1 mask commutes with the gate, so scale the [B, T, D] gate
# instead of the attention output
a2v_gate = a2v_gate * a2v_cross_attn_perturbation_mask
hidden_states = residual_gate_add(
hidden_states, a2v_attn_hidden_states, a2v_gate
)
@@ -1534,9 +1523,7 @@ class LTX2TransformerBlock(nn.Module):
context_replicated_prefix_len=video_memory_prefix_len,
)
if v2a_cross_attn_perturbation_mask is not None:
v2a_attn_hidden_states = (
v2a_attn_hidden_states * v2a_cross_attn_perturbation_mask
)
v2a_gate = v2a_gate * v2a_cross_attn_perturbation_mask
audio_hidden_states = residual_gate_add(
audio_hidden_states, v2a_attn_hidden_states, v2a_gate
)
@@ -0,0 +1,47 @@
import unittest
from types import SimpleNamespace
import torch
from sglang.multimodal_gen.runtime.models.dits.ltx_2 import LTX2TransformerBlock
def _reference_ada_values(table, batch_size, timestep, indices):
num = int(table.shape[0])
values = (
table[indices].unsqueeze(0).unsqueeze(0).to(dtype=timestep.dtype)
+ timestep.reshape(batch_size, timestep.shape[1], num, -1)[:, :, indices, :]
).unbind(dim=2)
return [t.squeeze(2) for t in values]
class TestLTX2AdaValuesLayout(unittest.TestCase):
def test_values_are_contiguous_and_bitwise_equal(self):
torch.manual_seed(0)
for table_dtype, tokens, indices in (
(torch.float32, 1, slice(0, 3)),
(torch.bfloat16, 1, slice(None)),
(torch.float32, 5, slice(6, 9)),
):
table = torch.randn(9, 16, dtype=table_dtype)
timestep = torch.randn(2, tokens, 9 * 16, dtype=torch.bfloat16)
actual = LTX2TransformerBlock.get_ada_values(
SimpleNamespace(), table, 2, timestep, indices
)
expected = _reference_ada_values(table, 2, timestep, indices)
self.assertEqual(len(actual), len(expected))
for a, e in zip(actual, expected):
self.assertTrue(a.is_contiguous())
self.assertTrue(torch.equal(a, e))
def test_zero_one_mask_commutes_with_the_gate(self):
torch.manual_seed(0)
x = torch.randn(2, 6, 16, dtype=torch.bfloat16)
attn = torch.randn(2, 6, 16, dtype=torch.bfloat16)
gate = torch.randn(2, 1, 16, dtype=torch.bfloat16)
mask = torch.tensor([1.0, 0.0], dtype=torch.bfloat16).view(2, 1, 1)
self.assertTrue(torch.equal(x + (attn * mask) * gate, x + attn * (gate * mask)))
if __name__ == "__main__":
unittest.main()