[Perf] diffusion: Project LTX-2 prompt K/V once per denoising loop (#46105)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 12:17:04 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 3b95120f00
commit f2c634961b
4 changed files with 307 additions and 11 deletions
@@ -4,6 +4,7 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Optional, Tuple, Union
import torch
@@ -138,6 +139,14 @@ def _ltx2_try_fused_qknorm_split_rope(
_LTX2_FUSED_ADA_VALUES_RUNTIME_DISABLED = False
@dataclass(slots=True)
class LTX2PromptKV:
"""One block's prompt cross-attention (K, V) with K normed, fixed for a denoising loop"""
video: tuple[torch.Tensor, torch.Tensor]
audio: tuple[torch.Tensor, torch.Tensor]
def adaln_embedding_coefficient(cross_attention_adaln: bool) -> int:
return ADALN_NUM_BASE_PARAMS + (
ADALN_NUM_CROSS_ATTN_PARAMS if cross_attention_adaln else 0
@@ -878,15 +887,20 @@ class LTX2Attention(nn.Module):
skip_sequence_parallel_override: bool = False,
gather_context_kv_for_sp: bool = False,
context_replicated_prefix_len: int = 0,
context_kv: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
gate_input = x
context_ = x if context is None else context
v, _ = self.to_v(context_)
if context_kv is None:
v, _ = self.to_v(context_)
else:
k, v = context_kv
use_attention = not all_perturbed
if use_attention:
q, _ = self.to_q(x)
k, _ = self.to_k(context_)
if context_kv is None:
k, _ = self.to_k(context_)
fused_qk = None
if pe is not None:
@@ -933,7 +947,8 @@ class LTX2Attention(nn.Module):
k_dtype = k.dtype
with torch.autocast(device_type=q.device.type, enabled=False):
q = self.q_norm(q).to(dtype=q_dtype)
k = self.k_norm(k).to(dtype=k_dtype)
if context_kv is None:
k = self.k_norm(k).to(dtype=k_dtype)
if pe is not None and cos.dim() == 3:
q = apply_interleaved_rotary_emb(q, (cos, sin))
@@ -1025,6 +1040,19 @@ class LTX2Attention(nn.Module):
return out_proj
def project_context_kv(
self, context: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""The normed K and the V that ``forward`` projects from an unrotated ``context``."""
v, _ = self.to_v(context)
k, _ = self.to_k(context)
if self.qk_norm:
assert self.k_norm is not None
k_dtype = k.dtype
with torch.autocast(device_type=k.device.type, enabled=False):
k = self.k_norm(k).to(dtype=k_dtype)
return k, v
def _slice_rope_for_tp(
self,
cos: torch.Tensor,
@@ -1306,6 +1334,7 @@ class LTX2TransformerBlock(nn.Module):
v2a_cross_attn_perturbation_mask: Optional[torch.Tensor] = None,
audio_replicated_for_sp: bool = False,
video_memory_prefix_len: int = 0,
prompt_kv: LTX2PromptKV | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
batch_size = hidden_states.size(0)
video_ada_values = _ltx2_try_fused_ada_values9(
@@ -1429,6 +1458,7 @@ class LTX2TransformerBlock(nn.Module):
norm_hidden_states,
context=encoder_hidden_states,
mask=encoder_attention_mask,
context_kv=None if prompt_kv is None else prompt_kv.video,
)
hidden_states = hidden_states + attn_hidden_states
@@ -1437,6 +1467,7 @@ class LTX2TransformerBlock(nn.Module):
norm_audio_hidden_states,
context=audio_encoder_hidden_states,
mask=audio_encoder_attention_mask,
context_kv=None if prompt_kv is None else prompt_kv.audio,
)
audio_hidden_states = audio_hidden_states + attn_audio_hidden_states
# 3. Audio-to-Video and Video-to-Audio Cross-Attention
@@ -1908,6 +1939,11 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.audio_out_channels = arch.audio_out_channels
self.timestep_scale_multiplier = arch.timestep_scale_multiplier
self.av_ca_timestep_scale_multiplier = arch.av_ca_timestep_scale_multiplier
# prompt K/V depend on the text alone unless adaLN modulates the prompt per step
self.caches_prompt_kv = not arch.cross_attention_adaln
# (video text shape, audio text shape) -> per-block prompt K/V; kept for the
# process lifetime so a captured forward keeps reading the same addresses
self._prompt_kv_buffers: dict[tuple[int, ...], list[LTX2PromptKV]] = {}
self.layer_names = ["transformer_blocks"]
@@ -1950,6 +1986,46 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
)
return video_timestep, audio_timestep_for_ca
@torch.no_grad()
def prepare_prompt_kv(
self,
encoder_hidden_states: torch.Tensor,
audio_encoder_hidden_states: torch.Tensor,
) -> tuple[int, ...]:
"""Project every block's prompt K/V into persistent buffers; returns the key ``forward`` reads them by.
Valid until the text or the DiT weights change, i.e. for one denoising loop.
"""
if self.caption_projection is not None:
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
if self.audio_caption_projection is not None:
audio_encoder_hidden_states = self.audio_caption_projection(
audio_encoder_hidden_states
)
key = (*encoder_hidden_states.shape, *audio_encoder_hidden_states.shape)
buffers = self._prompt_kv_buffers.get(key)
if buffers is None:
self._prompt_kv_buffers[key] = [
LTX2PromptKV(
video=block.attn2.project_context_kv(encoder_hidden_states),
audio=block.audio_attn2.project_context_kv(
audio_encoder_hidden_states
),
)
for block in self.transformer_blocks
]
return key
for block, prompt_kv in zip(self.transformer_blocks, buffers):
for dst, src in zip(
(*prompt_kv.video, *prompt_kv.audio),
(
*block.attn2.project_context_kv(encoder_hidden_states),
*block.audio_attn2.project_context_kv(audio_encoder_hidden_states),
),
):
dst.copy_(src)
return key
def forward(
self,
hidden_states: torch.Tensor,
@@ -1981,6 +2057,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
video_memory_prefix_len: int = 0,
late_layer_ratio: float = 1.0,
late_audio_self_attention_mask: Optional[torch.Tensor] = None,
cached_prompt_kv: tuple[int, ...] | None = None,
**kwargs,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
batch_size = hidden_states.size(0)
@@ -2153,13 +2230,17 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
batch_size, -1, temb_ca_audio_gate.shape[-1]
)
# 4. Prepare prompt embeddings
if self.caption_projection is not None:
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
if self.audio_caption_projection is not None:
audio_encoder_hidden_states = self.audio_caption_projection(
audio_encoder_hidden_states
)
# 4. Prepare prompt embeddings, or read the K/V prepare_prompt_kv projected from them
prompt_kvs = None
if cached_prompt_kv is not None:
prompt_kvs = self._prompt_kv_buffers[cached_prompt_kv]
else:
if self.caption_projection is not None:
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
if self.audio_caption_projection is not None:
audio_encoder_hidden_states = self.audio_caption_projection(
audio_encoder_hidden_states
)
if _is_npu:
# If the 'encoder_attention_mask' is provided and it is all ones,
@@ -2229,7 +2310,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
)
)
late_layer_start = int(len(self.transformer_blocks) * float(late_layer_ratio))
for block in self.transformer_blocks:
for block_pos, block in enumerate(self.transformer_blocks):
block_idx = getattr(block, "idx", -1)
video_self_attn_perturbation_mask = None
audio_self_attn_perturbation_mask = None
@@ -2300,6 +2381,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
v2a_cross_attn_perturbation_mask=v2a_cross_attn_perturbation_mask,
audio_replicated_for_sp=audio_replicated_for_sp,
video_memory_prefix_len=video_memory_prefix_len,
prompt_kv=None if prompt_kvs is None else prompt_kvs[block_pos],
)
# 6. Output layers
@@ -77,6 +77,8 @@ class LTX2DenoisingContext(DenoisingContext):
# fp32 copies of scheduler.sigmas taken once per loop, so steps never sync on them
sigmas_host: torch.Tensor | None = None
sigmas_device: torch.Tensor | None = None
# key of the prompt K/V the DiT projected for this loop (prepare_prompt_kv)
prompt_kv_key: tuple[int, ...] | None = None
@dataclass(slots=True)
@@ -1475,6 +1477,36 @@ class LTX2DenoisingStage(DenoisingStage):
):
yield
def _use_cached_ltx2_prompt_kv(
self,
ctx: LTX2DenoisingContext,
step: DenoisingStepState,
model_kwargs: dict[str, object],
) -> dict[str, object]:
"""Project the prompt K/V on the loop's first step and have every step read them.
The text and the DiT weights are fixed for one loop (the LoRA phase switches
before it), so the K/V each step would project are the same tensors.
"""
model = step.current_model
if (
not model.caches_prompt_kv
or model.layerwise_offload_managers
or self._cache_dit_enabled
):
return model_kwargs
if ctx.prompt_kv_key is None:
ctx.prompt_kv_key = model.prepare_prompt_kv(
model_kwargs["encoder_hidden_states"],
model_kwargs["audio_encoder_hidden_states"],
)
return {
**model_kwargs,
"encoder_hidden_states": None,
"audio_encoder_hidden_states": None,
"cached_prompt_kv": ctx.prompt_kv_key,
}
def _ltx2_call_current_model(
self,
ctx: "LTX2DenoisingContext",
@@ -1783,6 +1815,7 @@ class LTX2DenoisingStage(DenoisingStage):
model_kwargs["audio_encoder_attention_mask"] = (
repeated_attention_mask
)
model_kwargs = self._use_cached_ltx2_prompt_kv(ctx, step, model_kwargs)
with self._ltx2_model_forward_context(ctx, step):
model_video, model_audio = self._ltx2_call_current_model(
@@ -263,6 +263,8 @@ class TestLTX2GuidanceBCG(CustomTestCase):
stage._maybe_get_bcg_runner = Mock(return_value=runner)
stage._ltx2_model_forward_context = lambda *_: nullcontext()
stage.post_forward_for_ti2v_task = lambda batch, args, mask, latents, z: latents
# the stub DiT reads the prompt itself instead of projecting cached K/V
stage._use_cached_ltx2_prompt_kv = lambda ctx, step, kwargs: kwargs
args = SimpleNamespace(
enable_cfg_parallel=False,
pipeline_class_name=(
@@ -0,0 +1,179 @@
import unittest
from contextlib import ExitStack
from types import SimpleNamespace
from unittest.mock import Mock, patch
import torch
import sglang.multimodal_gen.runtime.models.dits.ltx_2 as ltx_2
from sglang.multimodal_gen.configs.models.dits.ltx_2 import LTX2ArchConfig, LTX2Config
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.attention import layer
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import SDPABackend
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
LTX2DenoisingStage,
)
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
ensure_distributed_env_defaults,
)
from sglang.test.test_utils import CustomTestCase
def _patch_attention_layers(stack: ExitStack) -> None:
for name, value in (
("get_attn_backend", SDPABackend),
("get_compute_dtype", torch.bfloat16),
("_resolve_sp_attention_mode", ("ulysses", False)),
("_maybe_install_backend_autotune", None),
):
stack.enter_context(patch.object(layer, name, return_value=value))
stack.enter_context(set_forward_context(0, None))
def _tiny_model(**arch_overrides) -> ltx_2.LTX2VideoTransformer3DModel:
if not model_parallel_is_initialized():
ensure_distributed_env_defaults()
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
torch.manual_seed(0)
arch = LTX2ArchConfig(
num_layers=2,
num_attention_heads=2,
attention_head_dim=32,
in_channels=16,
out_channels=16,
cross_attention_dim=64,
caption_channels=64,
audio_num_attention_heads=2,
audio_attention_head_dim=16,
audio_in_channels=16,
audio_out_channels=16,
audio_cross_attention_dim=32,
**arch_overrides,
)
model = ltx_2.LTX2VideoTransformer3DModel(
LTX2Config(arch_config=arch), hf_config={}
).to(torch.bfloat16)
with torch.no_grad():
for parameter in model.parameters():
parameter.normal_(std=0.05)
return model
def _forward_kwargs(text_seed: int) -> dict:
generator = torch.Generator().manual_seed(text_seed)
mask = torch.ones(1, 6, dtype=torch.int64)
mask[:, 4:] = 0
return dict(
hidden_states=torch.randn(1, 32, 16, dtype=torch.bfloat16),
audio_hidden_states=torch.randn(1, 5, 16, dtype=torch.bfloat16),
encoder_hidden_states=torch.randn(
1, 6, 64, generator=generator, dtype=torch.float32
).to(torch.bfloat16),
audio_encoder_hidden_states=torch.randn(
1, 6, 64, generator=generator, dtype=torch.float32
).to(torch.bfloat16),
timestep=torch.tensor([500.0]),
encoder_attention_mask=mask,
audio_encoder_attention_mask=mask,
num_frames=2,
height=4,
width=4,
audio_num_frames=5,
return_latents=False,
)
def _cached(model, kwargs: dict) -> dict:
key = model.prepare_prompt_kv(
kwargs["encoder_hidden_states"], kwargs["audio_encoder_hidden_states"]
)
return {
**kwargs,
"encoder_hidden_states": None,
"audio_encoder_hidden_states": None,
"cached_prompt_kv": key,
}
class TestLTX2PromptKVCache(CustomTestCase):
@torch.no_grad()
def test_cached_prompt_kv_is_bitwise_equal_to_per_step_projection(self):
with ExitStack() as stack:
_patch_attention_layers(stack)
model = _tiny_model()
kwargs = _forward_kwargs(text_seed=1)
expected = model(**kwargs)
actual = model(**_cached(model, kwargs))
for a, e in zip(actual, expected, strict=True):
self.assertTrue(torch.equal(a, e))
@torch.no_grad()
def test_new_text_is_written_into_the_same_buffers(self):
with ExitStack() as stack:
_patch_attention_layers(stack)
model = _tiny_model()
first = _cached(model, _forward_kwargs(text_seed=1))
buffers = model._prompt_kv_buffers[first["cached_prompt_kv"]]
pointers = [t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)]
second_kwargs = _forward_kwargs(text_seed=2)
second = _cached(model, second_kwargs)
self.assertEqual(second["cached_prompt_kv"], first["cached_prompt_kv"])
self.assertEqual(
[t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)],
pointers,
)
for a, e in zip(model(**second), model(**second_kwargs), strict=True):
self.assertTrue(torch.equal(a, e))
def test_prompt_modulated_cross_attention_does_not_cache(self):
self.assertTrue(_tiny_model().caches_prompt_kv)
self.assertFalse(_tiny_model(cross_attention_adaln=True).caches_prompt_kv)
class TestLTX2StagePromptKV(CustomTestCase):
def _model(self, *, caches: bool = True, offloaded: bool = False):
return SimpleNamespace(
caches_prompt_kv=caches,
layerwise_offload_managers=[object()] if offloaded else [],
prepare_prompt_kv=Mock(return_value=(1, 6, 64, 1, 6, 32)),
)
def _use(self, model, ctx, *, cache_dit: bool = False) -> dict:
stage = SimpleNamespace(_cache_dit_enabled=cache_dit)
kwargs = {
"encoder_hidden_states": torch.zeros(1),
"audio_encoder_hidden_states": torch.zeros(1),
}
return LTX2DenoisingStage._use_cached_ltx2_prompt_kv(
stage, ctx, SimpleNamespace(current_model=model), kwargs
)
def test_projects_once_per_loop(self):
model = self._model()
ctx = SimpleNamespace(prompt_kv_key=None)
for _ in range(3):
kwargs = self._use(model, ctx)
model.prepare_prompt_kv.assert_called_once()
self.assertEqual(kwargs["cached_prompt_kv"], (1, 6, 64, 1, 6, 32))
self.assertIsNone(kwargs["encoder_hidden_states"])
self.assertIsNone(kwargs["audio_encoder_hidden_states"])
def test_keeps_projecting_when_the_weights_or_text_may_move(self):
for model, cache_dit in (
(self._model(caches=False), False),
(self._model(offloaded=True), False),
(self._model(), True),
):
kwargs = self._use(
model, SimpleNamespace(prompt_kv_key=None), cache_dit=cache_dit
)
self.assertNotIn("cached_prompt_kv", kwargs)
model.prepare_prompt_kv.assert_not_called()
if __name__ == "__main__":
unittest.main()