[Perf] diffusion: Project LTX-2 prompt K/V once per denoising loop (#46105)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
3b95120f00
commit
f2c634961b
@@ -4,6 +4,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -138,6 +139,14 @@ def _ltx2_try_fused_qknorm_split_rope(
|
||||
_LTX2_FUSED_ADA_VALUES_RUNTIME_DISABLED = False
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LTX2PromptKV:
|
||||
"""One block's prompt cross-attention (K, V) with K normed, fixed for a denoising loop"""
|
||||
|
||||
video: tuple[torch.Tensor, torch.Tensor]
|
||||
audio: tuple[torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
def adaln_embedding_coefficient(cross_attention_adaln: bool) -> int:
|
||||
return ADALN_NUM_BASE_PARAMS + (
|
||||
ADALN_NUM_CROSS_ATTN_PARAMS if cross_attention_adaln else 0
|
||||
@@ -878,15 +887,20 @@ class LTX2Attention(nn.Module):
|
||||
skip_sequence_parallel_override: bool = False,
|
||||
gather_context_kv_for_sp: bool = False,
|
||||
context_replicated_prefix_len: int = 0,
|
||||
context_kv: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
gate_input = x
|
||||
context_ = x if context is None else context
|
||||
v, _ = self.to_v(context_)
|
||||
if context_kv is None:
|
||||
v, _ = self.to_v(context_)
|
||||
else:
|
||||
k, v = context_kv
|
||||
use_attention = not all_perturbed
|
||||
|
||||
if use_attention:
|
||||
q, _ = self.to_q(x)
|
||||
k, _ = self.to_k(context_)
|
||||
if context_kv is None:
|
||||
k, _ = self.to_k(context_)
|
||||
|
||||
fused_qk = None
|
||||
if pe is not None:
|
||||
@@ -933,7 +947,8 @@ class LTX2Attention(nn.Module):
|
||||
k_dtype = k.dtype
|
||||
with torch.autocast(device_type=q.device.type, enabled=False):
|
||||
q = self.q_norm(q).to(dtype=q_dtype)
|
||||
k = self.k_norm(k).to(dtype=k_dtype)
|
||||
if context_kv is None:
|
||||
k = self.k_norm(k).to(dtype=k_dtype)
|
||||
|
||||
if pe is not None and cos.dim() == 3:
|
||||
q = apply_interleaved_rotary_emb(q, (cos, sin))
|
||||
@@ -1025,6 +1040,19 @@ class LTX2Attention(nn.Module):
|
||||
|
||||
return out_proj
|
||||
|
||||
def project_context_kv(
|
||||
self, context: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""The normed K and the V that ``forward`` projects from an unrotated ``context``."""
|
||||
v, _ = self.to_v(context)
|
||||
k, _ = self.to_k(context)
|
||||
if self.qk_norm:
|
||||
assert self.k_norm is not None
|
||||
k_dtype = k.dtype
|
||||
with torch.autocast(device_type=k.device.type, enabled=False):
|
||||
k = self.k_norm(k).to(dtype=k_dtype)
|
||||
return k, v
|
||||
|
||||
def _slice_rope_for_tp(
|
||||
self,
|
||||
cos: torch.Tensor,
|
||||
@@ -1306,6 +1334,7 @@ class LTX2TransformerBlock(nn.Module):
|
||||
v2a_cross_attn_perturbation_mask: Optional[torch.Tensor] = None,
|
||||
audio_replicated_for_sp: bool = False,
|
||||
video_memory_prefix_len: int = 0,
|
||||
prompt_kv: LTX2PromptKV | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_size = hidden_states.size(0)
|
||||
video_ada_values = _ltx2_try_fused_ada_values9(
|
||||
@@ -1429,6 +1458,7 @@ class LTX2TransformerBlock(nn.Module):
|
||||
norm_hidden_states,
|
||||
context=encoder_hidden_states,
|
||||
mask=encoder_attention_mask,
|
||||
context_kv=None if prompt_kv is None else prompt_kv.video,
|
||||
)
|
||||
hidden_states = hidden_states + attn_hidden_states
|
||||
|
||||
@@ -1437,6 +1467,7 @@ class LTX2TransformerBlock(nn.Module):
|
||||
norm_audio_hidden_states,
|
||||
context=audio_encoder_hidden_states,
|
||||
mask=audio_encoder_attention_mask,
|
||||
context_kv=None if prompt_kv is None else prompt_kv.audio,
|
||||
)
|
||||
audio_hidden_states = audio_hidden_states + attn_audio_hidden_states
|
||||
# 3. Audio-to-Video and Video-to-Audio Cross-Attention
|
||||
@@ -1908,6 +1939,11 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
self.audio_out_channels = arch.audio_out_channels
|
||||
self.timestep_scale_multiplier = arch.timestep_scale_multiplier
|
||||
self.av_ca_timestep_scale_multiplier = arch.av_ca_timestep_scale_multiplier
|
||||
# prompt K/V depend on the text alone unless adaLN modulates the prompt per step
|
||||
self.caches_prompt_kv = not arch.cross_attention_adaln
|
||||
# (video text shape, audio text shape) -> per-block prompt K/V; kept for the
|
||||
# process lifetime so a captured forward keeps reading the same addresses
|
||||
self._prompt_kv_buffers: dict[tuple[int, ...], list[LTX2PromptKV]] = {}
|
||||
|
||||
self.layer_names = ["transformer_blocks"]
|
||||
|
||||
@@ -1950,6 +1986,46 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
)
|
||||
return video_timestep, audio_timestep_for_ca
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_prompt_kv(
|
||||
self,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
audio_encoder_hidden_states: torch.Tensor,
|
||||
) -> tuple[int, ...]:
|
||||
"""Project every block's prompt K/V into persistent buffers; returns the key ``forward`` reads them by.
|
||||
|
||||
Valid until the text or the DiT weights change, i.e. for one denoising loop.
|
||||
"""
|
||||
if self.caption_projection is not None:
|
||||
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
|
||||
if self.audio_caption_projection is not None:
|
||||
audio_encoder_hidden_states = self.audio_caption_projection(
|
||||
audio_encoder_hidden_states
|
||||
)
|
||||
key = (*encoder_hidden_states.shape, *audio_encoder_hidden_states.shape)
|
||||
buffers = self._prompt_kv_buffers.get(key)
|
||||
if buffers is None:
|
||||
self._prompt_kv_buffers[key] = [
|
||||
LTX2PromptKV(
|
||||
video=block.attn2.project_context_kv(encoder_hidden_states),
|
||||
audio=block.audio_attn2.project_context_kv(
|
||||
audio_encoder_hidden_states
|
||||
),
|
||||
)
|
||||
for block in self.transformer_blocks
|
||||
]
|
||||
return key
|
||||
for block, prompt_kv in zip(self.transformer_blocks, buffers):
|
||||
for dst, src in zip(
|
||||
(*prompt_kv.video, *prompt_kv.audio),
|
||||
(
|
||||
*block.attn2.project_context_kv(encoder_hidden_states),
|
||||
*block.audio_attn2.project_context_kv(audio_encoder_hidden_states),
|
||||
),
|
||||
):
|
||||
dst.copy_(src)
|
||||
return key
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1981,6 +2057,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
video_memory_prefix_len: int = 0,
|
||||
late_layer_ratio: float = 1.0,
|
||||
late_audio_self_attention_mask: Optional[torch.Tensor] = None,
|
||||
cached_prompt_kv: tuple[int, ...] | None = None,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
batch_size = hidden_states.size(0)
|
||||
@@ -2153,13 +2230,17 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
batch_size, -1, temb_ca_audio_gate.shape[-1]
|
||||
)
|
||||
|
||||
# 4. Prepare prompt embeddings
|
||||
if self.caption_projection is not None:
|
||||
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
|
||||
if self.audio_caption_projection is not None:
|
||||
audio_encoder_hidden_states = self.audio_caption_projection(
|
||||
audio_encoder_hidden_states
|
||||
)
|
||||
# 4. Prepare prompt embeddings, or read the K/V prepare_prompt_kv projected from them
|
||||
prompt_kvs = None
|
||||
if cached_prompt_kv is not None:
|
||||
prompt_kvs = self._prompt_kv_buffers[cached_prompt_kv]
|
||||
else:
|
||||
if self.caption_projection is not None:
|
||||
encoder_hidden_states = self.caption_projection(encoder_hidden_states)
|
||||
if self.audio_caption_projection is not None:
|
||||
audio_encoder_hidden_states = self.audio_caption_projection(
|
||||
audio_encoder_hidden_states
|
||||
)
|
||||
|
||||
if _is_npu:
|
||||
# If the 'encoder_attention_mask' is provided and it is all ones,
|
||||
@@ -2229,7 +2310,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
)
|
||||
)
|
||||
late_layer_start = int(len(self.transformer_blocks) * float(late_layer_ratio))
|
||||
for block in self.transformer_blocks:
|
||||
for block_pos, block in enumerate(self.transformer_blocks):
|
||||
block_idx = getattr(block, "idx", -1)
|
||||
video_self_attn_perturbation_mask = None
|
||||
audio_self_attn_perturbation_mask = None
|
||||
@@ -2300,6 +2381,7 @@ class LTX2VideoTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
v2a_cross_attn_perturbation_mask=v2a_cross_attn_perturbation_mask,
|
||||
audio_replicated_for_sp=audio_replicated_for_sp,
|
||||
video_memory_prefix_len=video_memory_prefix_len,
|
||||
prompt_kv=None if prompt_kvs is None else prompt_kvs[block_pos],
|
||||
)
|
||||
|
||||
# 6. Output layers
|
||||
|
||||
+33
@@ -77,6 +77,8 @@ class LTX2DenoisingContext(DenoisingContext):
|
||||
# fp32 copies of scheduler.sigmas taken once per loop, so steps never sync on them
|
||||
sigmas_host: torch.Tensor | None = None
|
||||
sigmas_device: torch.Tensor | None = None
|
||||
# key of the prompt K/V the DiT projected for this loop (prepare_prompt_kv)
|
||||
prompt_kv_key: tuple[int, ...] | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -1475,6 +1477,36 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
):
|
||||
yield
|
||||
|
||||
def _use_cached_ltx2_prompt_kv(
|
||||
self,
|
||||
ctx: LTX2DenoisingContext,
|
||||
step: DenoisingStepState,
|
||||
model_kwargs: dict[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Project the prompt K/V on the loop's first step and have every step read them.
|
||||
|
||||
The text and the DiT weights are fixed for one loop (the LoRA phase switches
|
||||
before it), so the K/V each step would project are the same tensors.
|
||||
"""
|
||||
model = step.current_model
|
||||
if (
|
||||
not model.caches_prompt_kv
|
||||
or model.layerwise_offload_managers
|
||||
or self._cache_dit_enabled
|
||||
):
|
||||
return model_kwargs
|
||||
if ctx.prompt_kv_key is None:
|
||||
ctx.prompt_kv_key = model.prepare_prompt_kv(
|
||||
model_kwargs["encoder_hidden_states"],
|
||||
model_kwargs["audio_encoder_hidden_states"],
|
||||
)
|
||||
return {
|
||||
**model_kwargs,
|
||||
"encoder_hidden_states": None,
|
||||
"audio_encoder_hidden_states": None,
|
||||
"cached_prompt_kv": ctx.prompt_kv_key,
|
||||
}
|
||||
|
||||
def _ltx2_call_current_model(
|
||||
self,
|
||||
ctx: "LTX2DenoisingContext",
|
||||
@@ -1783,6 +1815,7 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
model_kwargs["audio_encoder_attention_mask"] = (
|
||||
repeated_attention_mask
|
||||
)
|
||||
model_kwargs = self._use_cached_ltx2_prompt_kv(ctx, step, model_kwargs)
|
||||
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
model_video, model_audio = self._ltx2_call_current_model(
|
||||
|
||||
@@ -263,6 +263,8 @@ class TestLTX2GuidanceBCG(CustomTestCase):
|
||||
stage._maybe_get_bcg_runner = Mock(return_value=runner)
|
||||
stage._ltx2_model_forward_context = lambda *_: nullcontext()
|
||||
stage.post_forward_for_ti2v_task = lambda batch, args, mask, latents, z: latents
|
||||
# the stub DiT reads the prompt itself instead of projecting cached K/V
|
||||
stage._use_cached_ltx2_prompt_kv = lambda ctx, step, kwargs: kwargs
|
||||
args = SimpleNamespace(
|
||||
enable_cfg_parallel=False,
|
||||
pipeline_class_name=(
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
import sglang.multimodal_gen.runtime.models.dits.ltx_2 as ltx_2
|
||||
from sglang.multimodal_gen.configs.models.dits.ltx_2 import LTX2ArchConfig, LTX2Config
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention import layer
|
||||
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import SDPABackend
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
|
||||
LTX2DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _patch_attention_layers(stack: ExitStack) -> None:
|
||||
for name, value in (
|
||||
("get_attn_backend", SDPABackend),
|
||||
("get_compute_dtype", torch.bfloat16),
|
||||
("_resolve_sp_attention_mode", ("ulysses", False)),
|
||||
("_maybe_install_backend_autotune", None),
|
||||
):
|
||||
stack.enter_context(patch.object(layer, name, return_value=value))
|
||||
stack.enter_context(set_forward_context(0, None))
|
||||
|
||||
|
||||
def _tiny_model(**arch_overrides) -> ltx_2.LTX2VideoTransformer3DModel:
|
||||
if not model_parallel_is_initialized():
|
||||
ensure_distributed_env_defaults()
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
torch.manual_seed(0)
|
||||
arch = LTX2ArchConfig(
|
||||
num_layers=2,
|
||||
num_attention_heads=2,
|
||||
attention_head_dim=32,
|
||||
in_channels=16,
|
||||
out_channels=16,
|
||||
cross_attention_dim=64,
|
||||
caption_channels=64,
|
||||
audio_num_attention_heads=2,
|
||||
audio_attention_head_dim=16,
|
||||
audio_in_channels=16,
|
||||
audio_out_channels=16,
|
||||
audio_cross_attention_dim=32,
|
||||
**arch_overrides,
|
||||
)
|
||||
model = ltx_2.LTX2VideoTransformer3DModel(
|
||||
LTX2Config(arch_config=arch), hf_config={}
|
||||
).to(torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
for parameter in model.parameters():
|
||||
parameter.normal_(std=0.05)
|
||||
return model
|
||||
|
||||
|
||||
def _forward_kwargs(text_seed: int) -> dict:
|
||||
generator = torch.Generator().manual_seed(text_seed)
|
||||
mask = torch.ones(1, 6, dtype=torch.int64)
|
||||
mask[:, 4:] = 0
|
||||
return dict(
|
||||
hidden_states=torch.randn(1, 32, 16, dtype=torch.bfloat16),
|
||||
audio_hidden_states=torch.randn(1, 5, 16, dtype=torch.bfloat16),
|
||||
encoder_hidden_states=torch.randn(
|
||||
1, 6, 64, generator=generator, dtype=torch.float32
|
||||
).to(torch.bfloat16),
|
||||
audio_encoder_hidden_states=torch.randn(
|
||||
1, 6, 64, generator=generator, dtype=torch.float32
|
||||
).to(torch.bfloat16),
|
||||
timestep=torch.tensor([500.0]),
|
||||
encoder_attention_mask=mask,
|
||||
audio_encoder_attention_mask=mask,
|
||||
num_frames=2,
|
||||
height=4,
|
||||
width=4,
|
||||
audio_num_frames=5,
|
||||
return_latents=False,
|
||||
)
|
||||
|
||||
|
||||
def _cached(model, kwargs: dict) -> dict:
|
||||
key = model.prepare_prompt_kv(
|
||||
kwargs["encoder_hidden_states"], kwargs["audio_encoder_hidden_states"]
|
||||
)
|
||||
return {
|
||||
**kwargs,
|
||||
"encoder_hidden_states": None,
|
||||
"audio_encoder_hidden_states": None,
|
||||
"cached_prompt_kv": key,
|
||||
}
|
||||
|
||||
|
||||
class TestLTX2PromptKVCache(CustomTestCase):
|
||||
@torch.no_grad()
|
||||
def test_cached_prompt_kv_is_bitwise_equal_to_per_step_projection(self):
|
||||
with ExitStack() as stack:
|
||||
_patch_attention_layers(stack)
|
||||
model = _tiny_model()
|
||||
kwargs = _forward_kwargs(text_seed=1)
|
||||
expected = model(**kwargs)
|
||||
actual = model(**_cached(model, kwargs))
|
||||
for a, e in zip(actual, expected, strict=True):
|
||||
self.assertTrue(torch.equal(a, e))
|
||||
|
||||
@torch.no_grad()
|
||||
def test_new_text_is_written_into_the_same_buffers(self):
|
||||
with ExitStack() as stack:
|
||||
_patch_attention_layers(stack)
|
||||
model = _tiny_model()
|
||||
first = _cached(model, _forward_kwargs(text_seed=1))
|
||||
buffers = model._prompt_kv_buffers[first["cached_prompt_kv"]]
|
||||
pointers = [t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)]
|
||||
second_kwargs = _forward_kwargs(text_seed=2)
|
||||
second = _cached(model, second_kwargs)
|
||||
self.assertEqual(second["cached_prompt_kv"], first["cached_prompt_kv"])
|
||||
self.assertEqual(
|
||||
[t.data_ptr() for kv in buffers for t in (*kv.video, *kv.audio)],
|
||||
pointers,
|
||||
)
|
||||
for a, e in zip(model(**second), model(**second_kwargs), strict=True):
|
||||
self.assertTrue(torch.equal(a, e))
|
||||
|
||||
def test_prompt_modulated_cross_attention_does_not_cache(self):
|
||||
self.assertTrue(_tiny_model().caches_prompt_kv)
|
||||
self.assertFalse(_tiny_model(cross_attention_adaln=True).caches_prompt_kv)
|
||||
|
||||
|
||||
class TestLTX2StagePromptKV(CustomTestCase):
|
||||
def _model(self, *, caches: bool = True, offloaded: bool = False):
|
||||
return SimpleNamespace(
|
||||
caches_prompt_kv=caches,
|
||||
layerwise_offload_managers=[object()] if offloaded else [],
|
||||
prepare_prompt_kv=Mock(return_value=(1, 6, 64, 1, 6, 32)),
|
||||
)
|
||||
|
||||
def _use(self, model, ctx, *, cache_dit: bool = False) -> dict:
|
||||
stage = SimpleNamespace(_cache_dit_enabled=cache_dit)
|
||||
kwargs = {
|
||||
"encoder_hidden_states": torch.zeros(1),
|
||||
"audio_encoder_hidden_states": torch.zeros(1),
|
||||
}
|
||||
return LTX2DenoisingStage._use_cached_ltx2_prompt_kv(
|
||||
stage, ctx, SimpleNamespace(current_model=model), kwargs
|
||||
)
|
||||
|
||||
def test_projects_once_per_loop(self):
|
||||
model = self._model()
|
||||
ctx = SimpleNamespace(prompt_kv_key=None)
|
||||
for _ in range(3):
|
||||
kwargs = self._use(model, ctx)
|
||||
model.prepare_prompt_kv.assert_called_once()
|
||||
self.assertEqual(kwargs["cached_prompt_kv"], (1, 6, 64, 1, 6, 32))
|
||||
self.assertIsNone(kwargs["encoder_hidden_states"])
|
||||
self.assertIsNone(kwargs["audio_encoder_hidden_states"])
|
||||
|
||||
def test_keeps_projecting_when_the_weights_or_text_may_move(self):
|
||||
for model, cache_dit in (
|
||||
(self._model(caches=False), False),
|
||||
(self._model(offloaded=True), False),
|
||||
(self._model(), True),
|
||||
):
|
||||
kwargs = self._use(
|
||||
model, SimpleNamespace(prompt_kv_key=None), cache_dit=cache_dit
|
||||
)
|
||||
self.assertNotIn("cached_prompt_kv", kwargs)
|
||||
model.prepare_prompt_kv.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user