[Perf] diffusion: Cut LTX-2.3 per-block elementwise work (contiguous AdaLN values, gate-folded masks, vectorized head gate) (#46110)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
4e2d5b17ad
commit
bbc77e7db4
@@ -279,6 +279,12 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
|
||||
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
|
||||
_CUDA,
|
||||
),
|
||||
(
|
||||
"diffusion.head_gate_mul",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.head_gate_triton:head_gate_mul",
|
||||
_CUDA,
|
||||
),
|
||||
(
|
||||
"diffusion.fused_inplace_qknorm_rope",
|
||||
KernelBackend.JIT,
|
||||
@@ -714,6 +720,8 @@ _EXPORTS: dict[str, str] = {
|
||||
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
|
||||
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
|
||||
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
|
||||
"can_use_head_gate_mul": "modulate.head_gate_triton",
|
||||
"head_gate_mul": "modulate.head_gate_triton",
|
||||
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
|
||||
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
|
||||
"modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Per-head attention gate ``out * (2 * sigmoid(gate_logits))`` for LTX-2.3.
|
||||
|
||||
Eager broadcasts a ``[B, T, H, 1]`` gate over ``[B, T, H, D]``, which PyTorch
|
||||
runs as a non-vectorized strided kernel, plus a separate ``2 *`` pass. This
|
||||
kernel takes the eager ``sigmoid(gate_logits)`` and applies the doubling and
|
||||
the product in fp32 with one bf16 rounding, the same operations and rounding
|
||||
as eager, so the output is bit-identical.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _head_gate_mul_kernel(
|
||||
out_ptr,
|
||||
x_ptr,
|
||||
gate_ptr,
|
||||
hidden: tl.constexpr,
|
||||
num_heads: tl.constexpr,
|
||||
head_dim: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.arange(0, BLOCK_N)
|
||||
mask = cols < hidden
|
||||
x = tl.load(x_ptr + row * hidden + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
gate = tl.load(
|
||||
gate_ptr + row * num_heads + cols // head_dim, mask=mask, other=0.0
|
||||
).to(tl.float32)
|
||||
# eager rounds 2 * sigmoid to bf16 first; doubling a bf16 value is exact
|
||||
y = x * (gate * 2.0)
|
||||
tl.store(out_ptr + row * hidden + cols, y.to(tl.bfloat16), mask=mask)
|
||||
|
||||
|
||||
def can_use_head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> bool:
|
||||
"""``x``: contiguous ``[B, T, H, D]`` or ``[B, T, H * D]``; ``gate``: ``[B, T, H]``."""
|
||||
return (
|
||||
x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
and gate.dtype == torch.bfloat16
|
||||
and x.device == gate.device
|
||||
and x.is_contiguous()
|
||||
and gate.is_contiguous()
|
||||
and x.dim() in (3, 4)
|
||||
and gate.dim() == 3
|
||||
and x.shape[:2] == gate.shape[:2]
|
||||
and math.prod(x.shape[2:]) % gate.shape[-1] == 0
|
||||
and math.prod(x.shape[2:]) <= 16384
|
||||
)
|
||||
|
||||
|
||||
def head_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor:
|
||||
"""``x * (2 * gate)`` per head, returned as ``[B, T, H * D]``."""
|
||||
x = x.view(x.shape[0], x.shape[1], -1)
|
||||
hidden = int(x.shape[-1])
|
||||
num_heads = int(gate.shape[-1])
|
||||
out = torch.empty_like(x)
|
||||
rows = int(x.shape[0] * x.shape[1])
|
||||
if rows:
|
||||
_head_gate_mul_kernel[(rows,)](
|
||||
out,
|
||||
x,
|
||||
gate,
|
||||
hidden,
|
||||
num_heads,
|
||||
hidden // num_heads,
|
||||
BLOCK_N=triton.next_power_of_2(hidden),
|
||||
num_warps=4 if hidden <= 4096 else 8,
|
||||
)
|
||||
return out
|
||||
@@ -16,11 +16,13 @@ from sglang.kernels.ops.diffusion import (
|
||||
LTX2_RMS_NORM_MODULATE,
|
||||
BitExactFusionGate,
|
||||
can_use_fused_rmsnorm_modulation,
|
||||
can_use_head_gate_mul,
|
||||
can_use_linear_gelu,
|
||||
can_use_ltx2_qknorm_split_rope_cuda,
|
||||
can_use_modulate_scale_shift_cuda,
|
||||
fused_linear_gelu_tanh,
|
||||
fused_ltx2_rms_norm_modulate,
|
||||
head_gate_mul,
|
||||
ltx2_qknorm_split_rope_cuda,
|
||||
modulate_scale_shift_cuda,
|
||||
residual_gate_add,
|
||||
@@ -1009,10 +1011,14 @@ class LTX2Attention(nn.Module):
|
||||
|
||||
if self.to_gate_logits is not None:
|
||||
gate_logits, _ = self.to_gate_logits(gate_input)
|
||||
gate = torch.sigmoid(gate_logits)
|
||||
b, t = out.shape[:2]
|
||||
out = out.view(b, t, self.local_heads, self.dim_head)
|
||||
out = out * (2.0 * torch.sigmoid(gate_logits).unsqueeze(-1))
|
||||
out = out.view(b, t, self.local_heads * self.dim_head)
|
||||
if can_use_head_gate_mul(out, gate):
|
||||
out = head_gate_mul(out, gate)
|
||||
else:
|
||||
out = out.view(b, t, self.local_heads, self.dim_head)
|
||||
out = out * (2.0 * gate.unsqueeze(-1))
|
||||
out = out.view(b, t, self.local_heads * self.dim_head)
|
||||
|
||||
out_flat = out.flatten(2)
|
||||
out_proj, _ = self.to_out[0](out_flat)
|
||||
@@ -1250,16 +1256,21 @@ class LTX2TransformerBlock(nn.Module):
|
||||
indices: slice,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
num_ada_params = int(scale_shift_table.shape[0])
|
||||
ada_values = (
|
||||
scale_shift_table[indices]
|
||||
.unsqueeze(0)
|
||||
.unsqueeze(0)
|
||||
.to(device=timestep.device, dtype=timestep.dtype)
|
||||
+ timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[
|
||||
:, :, indices, :
|
||||
]
|
||||
).unbind(dim=2)
|
||||
return [t.squeeze(2) for t in ada_values]
|
||||
table = scale_shift_table[indices].to(
|
||||
device=timestep.device, dtype=timestep.dtype
|
||||
)
|
||||
temb = timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[
|
||||
:, :, indices, :
|
||||
]
|
||||
# value-major, so every returned [B, T, D] is contiguous and the modulate
|
||||
# kernels take it without a copy (a [B, T, k, D] slice is strided for B > 1)
|
||||
ada_values = torch.empty(
|
||||
(table.shape[0], *temb.shape[:2], temb.shape[-1]),
|
||||
dtype=temb.dtype,
|
||||
device=temb.device,
|
||||
)
|
||||
torch.add(table[:, None, None, :], temb.permute(2, 0, 1, 3), out=ada_values)
|
||||
return list(ada_values.unbind(0))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1433,62 +1444,40 @@ class LTX2TransformerBlock(nn.Module):
|
||||
norm_audio_hidden_states = self.rms_norm(audio_hidden_states, self.norm_eps)
|
||||
|
||||
# Compute combined ada params
|
||||
video_per_layer_ca_scale_shift = self.video_a2v_cross_attn_scale_shift_table[
|
||||
:4, :
|
||||
]
|
||||
video_per_layer_ca_gate = self.video_a2v_cross_attn_scale_shift_table[4:, :]
|
||||
|
||||
video_ca_scale_shift_table = (
|
||||
video_per_layer_ca_scale_shift[None, None, :, :].to(
|
||||
dtype=temb_ca_scale_shift.dtype, device=temb_ca_scale_shift.device
|
||||
)
|
||||
+ temb_ca_scale_shift.reshape(
|
||||
batch_size, temb_ca_scale_shift.shape[1], 4, -1
|
||||
)
|
||||
).unbind(dim=2)
|
||||
video_ca_gate = (
|
||||
video_per_layer_ca_gate[None, None, :, :].to(
|
||||
dtype=temb_ca_gate.dtype, device=temb_ca_gate.device
|
||||
)
|
||||
+ temb_ca_gate.reshape(batch_size, temb_ca_gate.shape[1], 1, -1)
|
||||
).unbind(dim=2)
|
||||
|
||||
(
|
||||
video_a2v_ca_scale,
|
||||
video_a2v_ca_shift,
|
||||
video_v2a_ca_scale,
|
||||
video_v2a_ca_shift,
|
||||
) = [t.squeeze(2) for t in video_ca_scale_shift_table]
|
||||
a2v_gate = video_ca_gate[0].squeeze(2)
|
||||
|
||||
audio_per_layer_ca_scale_shift = self.audio_a2v_cross_attn_scale_shift_table[
|
||||
:4, :
|
||||
]
|
||||
audio_per_layer_ca_gate = self.audio_a2v_cross_attn_scale_shift_table[4:, :]
|
||||
|
||||
audio_ca_scale_shift_table = (
|
||||
audio_per_layer_ca_scale_shift[None, None, :, :].to(
|
||||
dtype=temb_ca_audio_scale_shift.dtype,
|
||||
device=temb_ca_audio_scale_shift.device,
|
||||
)
|
||||
+ temb_ca_audio_scale_shift.reshape(
|
||||
batch_size, temb_ca_audio_scale_shift.shape[1], 4, -1
|
||||
)
|
||||
).unbind(dim=2)
|
||||
audio_ca_gate = (
|
||||
audio_per_layer_ca_gate[None, None, :, :].to(
|
||||
dtype=temb_ca_audio_gate.dtype, device=temb_ca_audio_gate.device
|
||||
)
|
||||
+ temb_ca_audio_gate.reshape(batch_size, temb_ca_audio_gate.shape[1], 1, -1)
|
||||
).unbind(dim=2)
|
||||
|
||||
) = self.get_ada_values(
|
||||
self.video_a2v_cross_attn_scale_shift_table[:4],
|
||||
batch_size,
|
||||
temb_ca_scale_shift,
|
||||
slice(None),
|
||||
)
|
||||
(a2v_gate,) = self.get_ada_values(
|
||||
self.video_a2v_cross_attn_scale_shift_table[4:],
|
||||
batch_size,
|
||||
temb_ca_gate,
|
||||
slice(None),
|
||||
)
|
||||
(
|
||||
audio_a2v_ca_scale,
|
||||
audio_a2v_ca_shift,
|
||||
audio_v2a_ca_scale,
|
||||
audio_v2a_ca_shift,
|
||||
) = [t.squeeze(2) for t in audio_ca_scale_shift_table]
|
||||
v2a_gate = audio_ca_gate[0].squeeze(2)
|
||||
) = self.get_ada_values(
|
||||
self.audio_a2v_cross_attn_scale_shift_table[:4],
|
||||
batch_size,
|
||||
temb_ca_audio_scale_shift,
|
||||
slice(None),
|
||||
)
|
||||
(v2a_gate,) = self.get_ada_values(
|
||||
self.audio_a2v_cross_attn_scale_shift_table[4:],
|
||||
batch_size,
|
||||
temb_ca_audio_gate,
|
||||
slice(None),
|
||||
)
|
||||
|
||||
# A2V
|
||||
mod_norm_hidden_states = _ltx2_modulate(
|
||||
@@ -1508,9 +1497,9 @@ class LTX2TransformerBlock(nn.Module):
|
||||
skip_sequence_parallel_override=audio_replicated_for_sp,
|
||||
)
|
||||
if a2v_cross_attn_perturbation_mask is not None:
|
||||
a2v_attn_hidden_states = (
|
||||
a2v_attn_hidden_states * a2v_cross_attn_perturbation_mask
|
||||
)
|
||||
# the 0/1 mask commutes with the gate, so scale the [B, T, D] gate
|
||||
# instead of the attention output
|
||||
a2v_gate = a2v_gate * a2v_cross_attn_perturbation_mask
|
||||
hidden_states = residual_gate_add(
|
||||
hidden_states, a2v_attn_hidden_states, a2v_gate
|
||||
)
|
||||
@@ -1534,9 +1523,7 @@ class LTX2TransformerBlock(nn.Module):
|
||||
context_replicated_prefix_len=video_memory_prefix_len,
|
||||
)
|
||||
if v2a_cross_attn_perturbation_mask is not None:
|
||||
v2a_attn_hidden_states = (
|
||||
v2a_attn_hidden_states * v2a_cross_attn_perturbation_mask
|
||||
)
|
||||
v2a_gate = v2a_gate * v2a_cross_attn_perturbation_mask
|
||||
audio_hidden_states = residual_gate_add(
|
||||
audio_hidden_states, v2a_attn_hidden_states, v2a_gate
|
||||
)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.dits.ltx_2 import LTX2TransformerBlock
|
||||
|
||||
|
||||
def _reference_ada_values(table, batch_size, timestep, indices):
|
||||
num = int(table.shape[0])
|
||||
values = (
|
||||
table[indices].unsqueeze(0).unsqueeze(0).to(dtype=timestep.dtype)
|
||||
+ timestep.reshape(batch_size, timestep.shape[1], num, -1)[:, :, indices, :]
|
||||
).unbind(dim=2)
|
||||
return [t.squeeze(2) for t in values]
|
||||
|
||||
|
||||
class TestLTX2AdaValuesLayout(unittest.TestCase):
|
||||
def test_values_are_contiguous_and_bitwise_equal(self):
|
||||
torch.manual_seed(0)
|
||||
for table_dtype, tokens, indices in (
|
||||
(torch.float32, 1, slice(0, 3)),
|
||||
(torch.bfloat16, 1, slice(None)),
|
||||
(torch.float32, 5, slice(6, 9)),
|
||||
):
|
||||
table = torch.randn(9, 16, dtype=table_dtype)
|
||||
timestep = torch.randn(2, tokens, 9 * 16, dtype=torch.bfloat16)
|
||||
actual = LTX2TransformerBlock.get_ada_values(
|
||||
SimpleNamespace(), table, 2, timestep, indices
|
||||
)
|
||||
expected = _reference_ada_values(table, 2, timestep, indices)
|
||||
self.assertEqual(len(actual), len(expected))
|
||||
for a, e in zip(actual, expected):
|
||||
self.assertTrue(a.is_contiguous())
|
||||
self.assertTrue(torch.equal(a, e))
|
||||
|
||||
def test_zero_one_mask_commutes_with_the_gate(self):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(2, 6, 16, dtype=torch.bfloat16)
|
||||
attn = torch.randn(2, 6, 16, dtype=torch.bfloat16)
|
||||
gate = torch.randn(2, 1, 16, dtype=torch.bfloat16)
|
||||
mask = torch.tensor([1.0, 0.0], dtype=torch.bfloat16).view(2, 1, 1)
|
||||
self.assertTrue(torch.equal(x + (attn * mask) * gate, x + attn * (gate * mask)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user