From f2c634961b9eae45d14525dae6c4154d66103a97 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 12:17:04 +0800 Subject: [PATCH] [Perf] diffusion: Project LTX-2 prompt K/V once per denoising loop (#46105) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/models/dits/ltx_2.py | 104 ++++++++-- .../model_specific_stages/ltx_2/denoising.py | 33 ++++ .../test/unit/test_ltx2_bcg_coords.py | 2 + .../test/unit/test_ltx2_prompt_kv_cache.py | 179 ++++++++++++++++++ 4 files changed, 307 insertions(+), 11 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_kv_cache.py diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index 23db3edc95..728ff47f66 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -4,6 +4,7 @@ from __future__ import annotations +from dataclasses import dataclass from typing import Any, Optional, Tuple, Union import torch @@ -138,6 +139,14 @@ def _ltx2_try_fused_qknorm_split_rope( _LTX2_FUSED_ADA_VALUES_RUNTIME_DISABLED = False +@dataclass(slots=True) +class LTX2PromptKV: + """One block's prompt cross-attention (K, V) with K normed, fixed for a denoising loop""" + + video: tuple[torch.Tensor, torch.Tensor] + audio: tuple[torch.Tensor, torch.Tensor] + + def adaln_embedding_coefficient(cross_attention_adaln: bool) -> int: return ADALN_NUM_BASE_PARAMS + ( ADALN_NUM_CROSS_ATTN_PARAMS if cross_attention_adaln else 0 @@ -878,15 +887,20 @@ class LTX2Attention(nn.Module): skip_sequence_parallel_override: bool = False, gather_context_kv_for_sp: bool = False, context_replicated_prefix_len: int = 0, + context_kv: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> torch.Tensor: gate_input = x context_ = x if context is None else context - v, _ = self.to_v(context_) + if context_kv is None: + v, _ = self.to_v(context_) + else: + k, v = context_kv use_attention = not all_perturbed if use_attention: q, _ = self.to_q(x) - k, _ = self.to_k(context_) + if context_kv is None: + k, _ = self.to_k(context_) fused_qk = None if pe is not None: @@ -933,7 +947,8 @@ class LTX2Attention(nn.Module): k_dtype = k.dtype with torch.autocast(device_type=q.device.type, enabled=False): q = self.q_norm(q).to(dtype=q_dtype) - k = self.k_norm(k).to(dtype=k_dtype) + if context_kv is None: + k = self.k_norm(k).to(dtype=k_dtype) if pe is not None and cos.dim() == 3: q = apply_interleaved_rotary_emb(q, (cos, sin)) @@ -1025,6 +1040,19 @@ class LTX2Attention(nn.Module): return out_proj + def project_context_kv( + self, context: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """The normed K and the V that ``forward`` projects from an unrotated ``context``.""" + v, _ = self.to_v(context) + k, _ = self.to_k(context) + if self.qk_norm: + assert self.k_norm is not None + k_dtype = k.dtype + with torch.autocast(device_type=k.device.type, enabled=False): + k = self.k_norm(k).to(dtype=k_dtype) + return k, v + def _slice_rope_for_tp( self, cos: torch.Tensor, @@ -1306,6 +1334,7 @@ class LTX2TransformerBlock(nn.Module): v2a_cross_attn_perturbation_mask: Optional[torch.Tensor] = None, audio_replicated_for_sp: bool = False, video_memory_prefix_len: int = 0, + prompt_kv: LTX2PromptKV | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: batch_size = hidden_states.size(0) video_ada_values = _ltx2_try_fused_ada_values9( @@ -1429,6 +1458,7 @@ class LTX2TransformerBlock(nn.Module): norm_hidden_states, context=encoder_hidden_states, mask=encoder_attention_mask, + context_kv=None if prompt_kv is None else prompt_kv.video, ) hidden_states = hidden_states + attn_hidden_states @@ -1437,6 +1467,7 @@ class LTX2TransformerBlock(nn.Module): norm_audio_hidden_states, context=audio_encoder_hidden_states, mask=audio_encoder_attention_mask, + context_kv=None if prompt_kv is None else prompt_kv.audio, ) audio_hidden_states = audio_hidden_states + attn_audio_hidden_states # 3. Audio-to-Video and Video-to-Audio Cross-Attention @@ -1908,6 +1939,11 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.audio_out_channels = arch.audio_out_channels self.timestep_scale_multiplier = arch.timestep_scale_multiplier self.av_ca_timestep_scale_multiplier = arch.av_ca_timestep_scale_multiplier + # prompt K/V depend on the text alone unless adaLN modulates the prompt per step + self.caches_prompt_kv = not arch.cross_attention_adaln + # (video text shape, audio text shape) -> per-block prompt K/V; kept for the + # process lifetime so a captured forward keeps reading the same addresses + self._prompt_kv_buffers: dict[tuple[int, ...], list[LTX2PromptKV]] = {} self.layer_names = ["transformer_blocks"] @@ -1950,6 +1986,46 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): ) return video_timestep, audio_timestep_for_ca + @torch.no_grad() + def prepare_prompt_kv( + self, + encoder_hidden_states: torch.Tensor, + audio_encoder_hidden_states: torch.Tensor, + ) -> tuple[int, ...]: + """Project every block's prompt K/V into persistent buffers; returns the key ``forward`` reads them by. + + Valid until the text or the DiT weights change, i.e. for one denoising loop. + """ + if self.caption_projection is not None: + encoder_hidden_states = self.caption_projection(encoder_hidden_states) + if self.audio_caption_projection is not None: + audio_encoder_hidden_states = self.audio_caption_projection( + audio_encoder_hidden_states + ) + key = (*encoder_hidden_states.shape, *audio_encoder_hidden_states.shape) + buffers = self._prompt_kv_buffers.get(key) + if buffers is None: + self._prompt_kv_buffers[key] = [ + LTX2PromptKV( + video=block.attn2.project_context_kv(encoder_hidden_states), + audio=block.audio_attn2.project_context_kv( + audio_encoder_hidden_states + ), + ) + for block in self.transformer_blocks + ] + return key + for block, prompt_kv in zip(self.transformer_blocks, buffers): + for dst, src in zip( + (*prompt_kv.video, *prompt_kv.audio), + ( + *block.attn2.project_context_kv(encoder_hidden_states), + *block.audio_attn2.project_context_kv(audio_encoder_hidden_states), + ), + ): + dst.copy_(src) + return key + def forward( self, hidden_states: torch.Tensor, @@ -1981,6 +2057,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): video_memory_prefix_len: int = 0, late_layer_ratio: float = 1.0, late_audio_self_attention_mask: Optional[torch.Tensor] = None, + cached_prompt_kv: tuple[int, ...] | None = None, **kwargs, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: batch_size = hidden_states.size(0) @@ -2153,13 +2230,17 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): batch_size, -1, temb_ca_audio_gate.shape[-1] ) - # 4. Prepare prompt embeddings - if self.caption_projection is not None: - encoder_hidden_states = self.caption_projection(encoder_hidden_states) - if self.audio_caption_projection is not None: - audio_encoder_hidden_states = self.audio_caption_projection( - audio_encoder_hidden_states - ) + # 4. Prepare prompt embeddings, or read the K/V prepare_prompt_kv projected from them + prompt_kvs = None + if cached_prompt_kv is not None: + prompt_kvs = self._prompt_kv_buffers[cached_prompt_kv] + else: + if self.caption_projection is not None: + encoder_hidden_states = self.caption_projection(encoder_hidden_states) + if self.audio_caption_projection is not None: + audio_encoder_hidden_states = self.audio_caption_projection( + audio_encoder_hidden_states + ) if _is_npu: # If the 'encoder_attention_mask' is provided and it is all ones, @@ -2229,7 +2310,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): ) ) late_layer_start = int(len(self.transformer_blocks) * float(late_layer_ratio)) - for block in self.transformer_blocks: + for block_pos, block in enumerate(self.transformer_blocks): block_idx = getattr(block, "idx", -1) video_self_attn_perturbation_mask = None audio_self_attn_perturbation_mask = None @@ -2300,6 +2381,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): v2a_cross_attn_perturbation_mask=v2a_cross_attn_perturbation_mask, audio_replicated_for_sp=audio_replicated_for_sp, video_memory_prefix_len=video_memory_prefix_len, + prompt_kv=None if prompt_kvs is None else prompt_kvs[block_pos], ) # 6. Output layers diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py index f8f086751b..d84f496a70 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py @@ -77,6 +77,8 @@ class LTX2DenoisingContext(DenoisingContext): # fp32 copies of scheduler.sigmas taken once per loop, so steps never sync on them sigmas_host: torch.Tensor | None = None sigmas_device: torch.Tensor | None = None + # key of the prompt K/V the DiT projected for this loop (prepare_prompt_kv) + prompt_kv_key: tuple[int, ...] | None = None @dataclass(slots=True) @@ -1475,6 +1477,36 @@ class LTX2DenoisingStage(DenoisingStage): ): yield + def _use_cached_ltx2_prompt_kv( + self, + ctx: LTX2DenoisingContext, + step: DenoisingStepState, + model_kwargs: dict[str, object], + ) -> dict[str, object]: + """Project the prompt K/V on the loop's first step and have every step read them. + + The text and the DiT weights are fixed for one loop (the LoRA phase switches + before it), so the K/V each step would project are the same tensors. + """ + model = step.current_model + if ( + not model.caches_prompt_kv + or model.layerwise_offload_managers + or self._cache_dit_enabled + ): + return model_kwargs + if ctx.prompt_kv_key is None: + ctx.prompt_kv_key = model.prepare_prompt_kv( + model_kwargs["encoder_hidden_states"], + model_kwargs["audio_encoder_hidden_states"], + ) + return { + **model_kwargs, + "encoder_hidden_states": None, + "audio_encoder_hidden_states": None, + "cached_prompt_kv": ctx.prompt_kv_key, + } + def _ltx2_call_current_model( self, ctx: "LTX2DenoisingContext", @@ -1783,6 +1815,7 @@ class LTX2DenoisingStage(DenoisingStage): model_kwargs["audio_encoder_attention_mask"] = ( repeated_attention_mask ) + model_kwargs = self._use_cached_ltx2_prompt_kv(ctx, step, model_kwargs) with self._ltx2_model_forward_context(ctx, step): model_video, model_audio = self._ltx2_call_current_model( diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py index 9b476be921..0cc7220eaa 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py @@ -263,6 +263,8 @@ class TestLTX2GuidanceBCG(CustomTestCase): stage._maybe_get_bcg_runner = Mock(return_value=runner) stage._ltx2_model_forward_context = lambda *_: nullcontext() stage.post_forward_for_ti2v_task = lambda batch, args, mask, latents, z: latents + # the stub DiT reads the prompt itself instead of projecting cached K/V + stage._use_cached_ltx2_prompt_kv = lambda ctx, step, kwargs: kwargs args = SimpleNamespace( enable_cfg_parallel=False, pipeline_class_name=( diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_kv_cache.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_kv_cache.py new file mode 100644 index 0000000000..783c438153 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_prompt_kv_cache.py @@ -0,0 +1,179 @@ +import unittest +from contextlib import ExitStack +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +import sglang.multimodal_gen.runtime.models.dits.ltx_2 as ltx_2 +from sglang.multimodal_gen.configs.models.dits.ltx_2 import LTX2ArchConfig, LTX2Config +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.layers.attention import layer +from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import SDPABackend +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import ( + LTX2DenoisingStage, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( + ensure_distributed_env_defaults, +) +from sglang.test.test_utils import CustomTestCase + + +def _patch_attention_layers(stack: ExitStack) -> None: + for name, value in ( + ("get_attn_backend", SDPABackend), + ("get_compute_dtype", torch.bfloat16), + ("_resolve_sp_attention_mode", ("ulysses", False)), + ("_maybe_install_backend_autotune", None), + ): + stack.enter_context(patch.object(layer, name, return_value=value)) + stack.enter_context(set_forward_context(0, None)) + + +def _tiny_model(**arch_overrides) -> ltx_2.LTX2VideoTransformer3DModel: + if not model_parallel_is_initialized(): + ensure_distributed_env_defaults() + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + torch.manual_seed(0) + arch = LTX2ArchConfig( + num_layers=2, + num_attention_heads=2, + attention_head_dim=32, + in_channels=16, + out_channels=16, + cross_attention_dim=64, + caption_channels=64, + audio_num_attention_heads=2, + audio_attention_head_dim=16, + audio_in_channels=16, + audio_out_channels=16, + audio_cross_attention_dim=32, + **arch_overrides, + ) + model = ltx_2.LTX2VideoTransformer3DModel( + LTX2Config(arch_config=arch), hf_config={} + ).to(torch.bfloat16) + with torch.no_grad(): + for parameter in model.parameters(): + parameter.normal_(std=0.05) + return model + + +def _forward_kwargs(text_seed: int) -> dict: + generator = torch.Generator().manual_seed(text_seed) + mask = torch.ones(1, 6, dtype=torch.int64) + mask[:, 4:] = 0 + return dict( + hidden_states=torch.randn(1, 32, 16, dtype=torch.bfloat16), + audio_hidden_states=torch.randn(1, 5, 16, dtype=torch.bfloat16), + encoder_hidden_states=torch.randn( + 1, 6, 64, generator=generator, dtype=torch.float32 + ).to(torch.bfloat16), + audio_encoder_hidden_states=torch.randn( + 1, 6, 64, generator=generator, dtype=torch.float32 + ).to(torch.bfloat16), + timestep=torch.tensor([500.0]), + encoder_attention_mask=mask, + audio_encoder_attention_mask=mask, + num_frames=2, + height=4, + width=4, + audio_num_frames=5, + return_latents=False, + ) + + +def _cached(model, kwargs: dict) -> dict: + key = model.prepare_prompt_kv( + kwargs["encoder_hidden_states"], kwargs["audio_encoder_hidden_states"] + ) + return { + **kwargs, + "encoder_hidden_states": None, + "audio_encoder_hidden_states": None, + "cached_prompt_kv": key, + } + + +class TestLTX2PromptKVCache(CustomTestCase): + @torch.no_grad() + def test_cached_prompt_kv_is_bitwise_equal_to_per_step_projection(self): + with ExitStack() as stack: + _patch_attention_layers(stack) + model = _tiny_model() + kwargs = _forward_kwargs(text_seed=1) + expected = model(**kwargs) + actual = model(**_cached(model, kwargs)) + for a, e in zip(actual, expected, strict=True): + self.assertTrue(torch.equal(a, e)) + + @torch.no_grad() + def test_new_text_is_written_into_the_same_buffers(self): + with ExitStack() as stack: + _patch_attention_layers(stack) + model = _tiny_model() + first = _cached(model, _forward_kwargs(text_seed=1)) + buffers = model._prompt_kv_buffers[first["cached_prompt_kv"]] + pointers = [t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)] + second_kwargs = _forward_kwargs(text_seed=2) + second = _cached(model, second_kwargs) + self.assertEqual(second["cached_prompt_kv"], first["cached_prompt_kv"]) + self.assertEqual( + [t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)], + pointers, + ) + for a, e in zip(model(**second), model(**second_kwargs), strict=True): + self.assertTrue(torch.equal(a, e)) + + def test_prompt_modulated_cross_attention_does_not_cache(self): + self.assertTrue(_tiny_model().caches_prompt_kv) + self.assertFalse(_tiny_model(cross_attention_adaln=True).caches_prompt_kv) + + +class TestLTX2StagePromptKV(CustomTestCase): + def _model(self, *, caches: bool = True, offloaded: bool = False): + return SimpleNamespace( + caches_prompt_kv=caches, + layerwise_offload_managers=[object()] if offloaded else [], + prepare_prompt_kv=Mock(return_value=(1, 6, 64, 1, 6, 32)), + ) + + def _use(self, model, ctx, *, cache_dit: bool = False) -> dict: + stage = SimpleNamespace(_cache_dit_enabled=cache_dit) + kwargs = { + "encoder_hidden_states": torch.zeros(1), + "audio_encoder_hidden_states": torch.zeros(1), + } + return LTX2DenoisingStage._use_cached_ltx2_prompt_kv( + stage, ctx, SimpleNamespace(current_model=model), kwargs + ) + + def test_projects_once_per_loop(self): + model = self._model() + ctx = SimpleNamespace(prompt_kv_key=None) + for _ in range(3): + kwargs = self._use(model, ctx) + model.prepare_prompt_kv.assert_called_once() + self.assertEqual(kwargs["cached_prompt_kv"], (1, 6, 64, 1, 6, 32)) + self.assertIsNone(kwargs["encoder_hidden_states"]) + self.assertIsNone(kwargs["audio_encoder_hidden_states"]) + + def test_keeps_projecting_when_the_weights_or_text_may_move(self): + for model, cache_dit in ( + (self._model(caches=False), False), + (self._model(offloaded=True), False), + (self._model(), True), + ): + kwargs = self._use( + model, SimpleNamespace(prompt_kv_key=None), cache_dit=cache_dit + ) + self.assertNotIn("cached_prompt_kv", kwargs) + model.prepare_prompt_kv.assert_not_called() + + +if __name__ == "__main__": + unittest.main()