diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py index f6594bc7af..47a9727b67 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py @@ -1,4 +1,5 @@ import math +from collections import OrderedDict from contextlib import contextmanager from dataclasses import dataclass, field @@ -179,6 +180,10 @@ class LTX2DenoisingStage(DenoisingStage): self.sampler_name = sampler_name # set per request by _prepare_denoising_loop before the cache-dit hook self._disable_cache_dit_for_request = False + self._ltx2_coords_cache: OrderedDict[ + tuple, tuple[torch.Tensor | None, torch.Tensor | None] + ] = OrderedDict() + self._ltx2_coords_cache_max_entries = 4 def _scheduler_step_kwargs(self, batch: Req, scheduler) -> dict: return self.prepare_extra_func_kwargs( @@ -1045,19 +1050,45 @@ class LTX2DenoisingStage(DenoisingStage): f"{audio_latent_model_input.ndim}, shape={tuple(audio_latent_model_input.shape)}" ) - def _prepare_ltx2_model_inputs( + def _get_ltx2_rope_coords( self, ctx: LTX2DenoisingContext, step: DenoisingStepState, batch: Req, server_args: ServerArgs, - sigma: torch.Tensor, - ) -> LTX2ModelInputs: - latent_model_input = ctx.latents.to(ctx.target_dtype) - audio_latent_model_input = ctx.audio_latents.to(ctx.target_dtype) - audio_num_frames_latent = self._get_audio_num_frames_latent( - audio_latent_model_input + latent_model_input: torch.Tensor, + audio_latent_model_input: torch.Tensor, + audio_num_frames_latent: int, + ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + video_sp_start = ( + int(batch.sp_video_start_frame) if batch.did_sp_shard_latents else None ) + audio_sp_start = ( + int(batch.sp_audio_start_frame) + if batch.did_sp_shard_audio_latents + else None + ) + key = ( + id(step.current_model), + id(server_args.pipeline_config), + latent_model_input.device, + audio_latent_model_input.device, + tuple(latent_model_input.shape), + tuple(audio_latent_model_input.shape), + ctx.latent_num_frames_for_model, + ctx.latent_height, + ctx.latent_width, + float(batch.fps), + audio_num_frames_latent, + ctx.use_ltx23_legacy_one_stage, + server_args.enable_breakable_cuda_graph, + video_sp_start, + audio_sp_start, + ) + cached = self._ltx2_coords_cache.get(key) + if cached is not None: + self._ltx2_coords_cache.move_to_end(key) + return cached video_coords = None audio_coords = None @@ -1076,7 +1107,7 @@ class LTX2DenoisingStage(DenoisingStage): audio_latent_model_input, num_frames=audio_num_frames_latent, ) - video_coords, audio_coords = _prepare_ltx2_rope_coords_for_bcg( + coords = _prepare_ltx2_rope_coords_for_bcg( enabled=server_args.enable_breakable_cuda_graph, current_model=step.current_model, latent_model_input=latent_model_input, @@ -1089,6 +1120,34 @@ class LTX2DenoisingStage(DenoisingStage): audio_num_frames=audio_num_frames_latent, fps=batch.fps, ) + self._ltx2_coords_cache[key] = coords + if len(self._ltx2_coords_cache) > self._ltx2_coords_cache_max_entries: + self._ltx2_coords_cache.popitem(last=False) + return coords + + def _prepare_ltx2_model_inputs( + self, + ctx: LTX2DenoisingContext, + step: DenoisingStepState, + batch: Req, + server_args: ServerArgs, + sigma: torch.Tensor, + ) -> LTX2ModelInputs: + latent_model_input = ctx.latents.to(ctx.target_dtype) + audio_latent_model_input = ctx.audio_latents.to(ctx.target_dtype) + audio_num_frames_latent = self._get_audio_num_frames_latent( + audio_latent_model_input + ) + + video_coords, audio_coords = self._get_ltx2_rope_coords( + ctx, + step, + batch, + server_args, + latent_model_input=latent_model_input, + audio_latent_model_input=audio_latent_model_input, + audio_num_frames_latent=audio_num_frames_latent, + ) batch_size = int(latent_model_input.shape[0]) use_raw_sigma_timestep = ctx.use_ltx23_hq_timestep_semantics diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py index b433b83477..241349ff3f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_bcg_coords.py @@ -1,10 +1,12 @@ import unittest +from collections import OrderedDict from types import SimpleNamespace from unittest.mock import Mock import torch from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import ( + LTX2DenoisingStage, _prepare_ltx2_rope_coords_for_bcg, ) from sglang.test.test_utils import CustomTestCase @@ -82,6 +84,55 @@ class TestLTX2BCGCoords(CustomTestCase): self.model.rope.prepare_video_coords.assert_not_called() self.model.audio_rope.prepare_audio_coords.assert_not_called() + def test_stage_reuses_coords_across_denoising_steps(self): + stage = object.__new__(LTX2DenoisingStage) + stage._ltx2_coords_cache = OrderedDict() + stage._ltx2_coords_cache_max_entries = 4 + pipeline_config = SimpleNamespace( + prepare_video_rope_coords_for_sp=Mock(return_value=self.video_result), + prepare_audio_rope_coords_for_sp=Mock(return_value=self.audio_result), + ) + ctx = SimpleNamespace( + latent_num_frames_for_model=16, + latent_height=32, + latent_width=48, + use_ltx23_legacy_one_stage=False, + ) + step = SimpleNamespace(current_model=self.model) + batch = SimpleNamespace( + fps=24, + did_sp_shard_latents=False, + did_sp_shard_audio_latents=False, + ) + server_args = SimpleNamespace( + pipeline_config=pipeline_config, + enable_breakable_cuda_graph=False, + ) + + first = stage._get_ltx2_rope_coords( + ctx, + step, + batch, + server_args, + self.video_latents, + self.audio_latents, + audio_num_frames_latent=126, + ) + second = stage._get_ltx2_rope_coords( + ctx, + step, + batch, + server_args, + self.video_latents, + self.audio_latents, + audio_num_frames_latent=126, + ) + + self.assertIs(first[0], second[0]) + self.assertIs(first[1], second[1]) + pipeline_config.prepare_video_rope_coords_for_sp.assert_called_once() + pipeline_config.prepare_audio_rope_coords_for_sp.assert_called_once() + if __name__ == "__main__": unittest.main()