From d0a098b2e08c080bd735cdee97479f9f39148a81 Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:36:01 +0800 Subject: [PATCH] [diffusion] perf: split and batch LTX-2.3 one-stage guidance passes under CFG parallel (#43684) Co-authored-by: Mick Qian --- .../stages/encode_while_decoding.py | 1 + .../model_specific_stages/joy_echo/setup.py | 1 + .../ltx_2/decoding_av.py | 12 +- .../model_specific_stages/ltx_2/denoising.py | 374 ++++++++---------- .../unit/test_ltx2_cfg_parallel_guidance.py | 108 +++++ .../test/unit/test_ltx_vae_on_frames.py | 91 ++++- 6 files changed, 357 insertions(+), 230 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_ltx2_cfg_parallel_guidance.py diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py index bbf47e9dcc..ad64b850d0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py @@ -33,6 +33,7 @@ def can_encode_while_decoding( batch.save_output and batch.return_file_paths_only and not batch.return_raw_frames + and not (batch.extra or {}).get("retain_decoded_frames", False) and batch.data_type == DataType.VIDEO and not batch.enable_frame_interpolation and not batch.enable_upscaling diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/joy_echo/setup.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/joy_echo/setup.py index 5d91c67619..0d0207f2a3 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/joy_echo/setup.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/joy_echo/setup.py @@ -48,6 +48,7 @@ class JoyEchoMultishotSetupStage(PipelineStage): logger.info("JoyEcho memory bank reset for new multi-shot session") def forward(self, batch: Req, server_args: ServerArgs) -> Req: + batch.extra["retain_decoded_frames"] = batch.enable_memory_bank if not batch.enable_memory_bank: return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py index 14220d84df..ba5910cb36 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py @@ -3,7 +3,11 @@ from collections.abc import Callable import torch from diffusers.video_processor import VideoProcessor -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_replica_group, + model_parallel_is_initialized, +) from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) @@ -152,7 +156,11 @@ class LTX2AVDecodingStage(DecodingStage): raise if output_paths is not None: output_batch.output_file_paths = output_paths - else: + elif ( + batch.extra.get("retain_decoded_frames", False) + or not model_parallel_is_initialized() + or get_replica_group().rank_in_group == 0 + ): output_batch.output = self.video_processor.postprocess_video( video, output_type="np" ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py index 6c352b797d..bb5b1b63cc 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising.py @@ -1947,20 +1947,28 @@ class LTX2DenoisingStage(DenoisingStage): float(stage1_guider_params["video_modality_scale"]) != 1.0 or float(stage1_guider_params["audio_modality_scale"]) != 1.0 ) - stage1_cfg_parallel = ( - server_args.enable_cfg_parallel and not ctx.use_ltx23_legacy_one_stage - ) + # Legacy LTX-2.3 one-stage runs the same guidance passes, so CFG + # parallel splits them across ranks like the other LTX-2 paths. + stage1_cfg_parallel = server_args.enable_cfg_parallel stage1_cfg_rank = ( get_classifier_free_guidance_rank() if stage1_cfg_parallel else 0 ) stage1_cfg_world_size = ( get_classifier_free_guidance_world_size() if stage1_cfg_parallel else 1 ) + # Legacy LTX-2.3 one-stage batches its guidance passes into one forward + # (perturbation masks are exact per item; only GEMM/attention rounding + # can move with the batch size). quality="exact" keeps the reference + # one-pass-per-forward schedule, which is bit-identical to it. + use_legacy_exact_passes = ctx.use_ltx23_legacy_one_stage and not ( + quality_allows(batch.quality, "lossless") + ) # NOTE: this flag must be identical across all SP ranks so that every # rank executes the same number of model-forward calls (each of which # contains NCCL collectives). use_split_stage1_guided_passes = ( server_args.pipeline_class_name == "LTX2TwoStageHQPipeline" + or use_legacy_exact_passes or ( is_ltx2_two_stage_pipeline_name(server_args.pipeline_class_name) and int(getattr(batch, "ltx2_num_image_tokens", 0)) > 0 @@ -1980,6 +1988,7 @@ class LTX2DenoisingStage(DenoisingStage): # perturbation_configs: one config dict per expanded item. use_split_pass_kwargs = ( server_args.pipeline_class_name == "LTX2TwoStageHQPipeline" + or use_legacy_exact_passes ) skip_v2a_cross_attn_for_video_gt = bool( batch.extra.get("ltx2_skip_v2a_cross_attn_for_video_gt", False) @@ -2005,240 +2014,165 @@ class LTX2DenoisingStage(DenoisingStage): ctx, batch, model_inputs_local ) - if ctx.use_ltx23_legacy_one_stage: - with self._ltx2_model_forward_context(ctx, step): - v_pos, a_v_pos = step.current_model( - **self._build_ltx2_model_kwargs( - ctx, - base_model_kwargs_local, - encoder_hidden_states=encoder_hidden_states, - audio_encoder_hidden_states=audio_encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - disable_v2a_cross_attn=( - skip_v2a_cross_attn_for_video_gt - ), - ) - ) - v_neg, a_v_neg = step.current_model( - **self._build_ltx2_model_kwargs( - ctx, - base_model_kwargs_local, - encoder_hidden_states=negative_encoder_hidden_states, - audio_encoder_hidden_states=negative_audio_encoder_hidden_states, - encoder_attention_mask=negative_encoder_attention_mask, - disable_v2a_cross_attn=( - skip_v2a_cross_attn_for_video_gt - ), - ) - ) - - v_pos = v_pos.float() - a_v_pos = a_v_pos.float() - v_neg = v_neg.float() - a_v_neg = a_v_neg.float() - - v_ptb = None - a_v_ptb = None - if need_perturbed: - with self._ltx2_model_forward_context(ctx, step): - v_ptb, a_v_ptb = step.current_model( - **self._build_ltx2_model_kwargs( - ctx, - base_model_kwargs_local, - encoder_hidden_states=encoder_hidden_states, - audio_encoder_hidden_states=audio_encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - skip_video_self_attn_blocks=tuple( - stage1_guider_params["video_stg_blocks"] - ), - skip_audio_self_attn_blocks=tuple( - stage1_guider_params["audio_stg_blocks"] - ), - disable_v2a_cross_attn=( - skip_v2a_cross_attn_for_video_gt - ), - ) - ) - v_ptb = v_ptb.float() - a_v_ptb = a_v_ptb.float() - - v_mod = None - a_v_mod = None - if need_modality: - with self._ltx2_model_forward_context(ctx, step): - v_mod, a_v_mod = step.current_model( - **self._build_ltx2_model_kwargs( - ctx, - base_model_kwargs_local, - encoder_hidden_states=encoder_hidden_states, - audio_encoder_hidden_states=audio_encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - disable_a2v_cross_attn=True, - disable_v2a_cross_attn=True, - ) - ) - v_mod = v_mod.float() - a_v_mod = a_v_mod.float() - else: - pass_specs: list[LTX2GuidancePassSpec] = [ + pass_specs: list[LTX2GuidancePassSpec] = [ + LTX2GuidancePassSpec( + name="cond", + encoder_hidden_states=encoder_hidden_states, + audio_encoder_hidden_states=audio_encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt, + ), + LTX2GuidancePassSpec( + name="neg", + encoder_hidden_states=negative_encoder_hidden_states, + audio_encoder_hidden_states=negative_audio_encoder_hidden_states, + encoder_attention_mask=negative_encoder_attention_mask, + disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt, + ), + ] + if need_perturbed: + pass_specs.append( LTX2GuidancePassSpec( - name="cond", + name="perturbed", encoder_hidden_states=encoder_hidden_states, audio_encoder_hidden_states=audio_encoder_hidden_states, encoder_attention_mask=encoder_attention_mask, + skip_video_self_attn_blocks=tuple( + stage1_guider_params["video_stg_blocks"] + ), + skip_audio_self_attn_blocks=tuple( + stage1_guider_params["audio_stg_blocks"] + ), disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt, - ), + ) + ) + if need_modality: + pass_specs.append( LTX2GuidancePassSpec( - name="neg", - encoder_hidden_states=negative_encoder_hidden_states, - audio_encoder_hidden_states=negative_audio_encoder_hidden_states, - encoder_attention_mask=negative_encoder_attention_mask, - disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt, - ), + name="modality", + encoder_hidden_states=encoder_hidden_states, + audio_encoder_hidden_states=audio_encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + disable_a2v_cross_attn=True, + disable_v2a_cross_attn=True, + ) + ) + + execution_pass_specs = ( + [ + pass_spec + for index, pass_spec in enumerate(pass_specs) + if index % stage1_cfg_world_size == stage1_cfg_rank ] - if need_perturbed: - pass_specs.append( - LTX2GuidancePassSpec( - name="perturbed", - encoder_hidden_states=encoder_hidden_states, - audio_encoder_hidden_states=audio_encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - skip_video_self_attn_blocks=tuple( - stage1_guider_params["video_stg_blocks"] - ), - skip_audio_self_attn_blocks=tuple( - stage1_guider_params["audio_stg_blocks"] - ), - disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt, - ) - ) - if need_modality: - pass_specs.append( - LTX2GuidancePassSpec( - name="modality", - encoder_hidden_states=encoder_hidden_states, - audio_encoder_hidden_states=audio_encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - disable_a2v_cross_attn=True, - disable_v2a_cross_attn=True, - ) - ) - - execution_pass_specs = ( + if stage1_cfg_parallel + else pass_specs + ) + num_execution_passes = len(execution_pass_specs) + if num_execution_passes == 0: + raise ValueError( + "LTX2 stage-1 CFG parallel degree exceeds guidance pass count." + ) + expanded_batch_size = batch_size_local * num_execution_passes + batched_model_kwargs = self._repeat_ltx2_model_kwargs_batch( + base_model_kwargs_local, expanded_batch_size + ) + batched_model_kwargs = self._build_ltx2_model_kwargs( + ctx, + batched_model_kwargs, + encoder_hidden_states=torch.cat( [ - pass_spec - for index, pass_spec in enumerate(pass_specs) - if index % stage1_cfg_world_size == stage1_cfg_rank - ] - if stage1_cfg_parallel - else pass_specs - ) - num_execution_passes = len(execution_pass_specs) - if num_execution_passes == 0: - raise ValueError( - "LTX2 stage-1 CFG parallel degree exceeds guidance pass count." - ) - expanded_batch_size = batch_size_local * num_execution_passes - batched_model_kwargs = self._repeat_ltx2_model_kwargs_batch( - base_model_kwargs_local, expanded_batch_size - ) - batched_model_kwargs = self._build_ltx2_model_kwargs( - ctx, - batched_model_kwargs, - encoder_hidden_states=torch.cat( - [ - pass_spec.encoder_hidden_states - for pass_spec in execution_pass_specs - ], - dim=0, - ), - audio_encoder_hidden_states=torch.cat( - [ - pass_spec.audio_encoder_hidden_states - for pass_spec in execution_pass_specs - ], - dim=0, - ), - encoder_attention_mask=self._cat_or_none( - [ - pass_spec.encoder_attention_mask - for pass_spec in execution_pass_specs - ] - ), - ) - if use_split_stage1_guided_passes: - split_sizes = [1] * expanded_batch_size - split_pass_specs = tuple( - pass_spec + pass_spec.encoder_hidden_states for pass_spec in execution_pass_specs - for _ in range(batch_size_local) + ], + dim=0, + ), + audio_encoder_hidden_states=torch.cat( + [ + pass_spec.audio_encoder_hidden_states + for pass_spec in execution_pass_specs + ], + dim=0, + ), + encoder_attention_mask=self._cat_or_none( + [ + pass_spec.encoder_attention_mask + for pass_spec in execution_pass_specs + ] + ), + ) + if use_split_stage1_guided_passes: + split_sizes = [1] * expanded_batch_size + split_pass_specs = tuple( + pass_spec + for pass_spec in execution_pass_specs + for _ in range(batch_size_local) + ) + split_perturbation_configs = ( + () + if use_split_pass_kwargs + else self._build_ltx2_guidance_perturbation_configs( + execution_pass_specs, batch_size_local ) - split_perturbation_configs = ( - () - if use_split_pass_kwargs - else self._build_ltx2_guidance_perturbation_configs( - execution_pass_specs, batch_size_local + ) + batched_video_chunks = [] + batched_audio_chunks = [] + with self._ltx2_model_forward_context(ctx, step): + for index, (model_kwargs_chunk, pass_spec) in enumerate( + zip( + self._split_ltx2_model_kwargs( + batched_model_kwargs, split_sizes + ), + split_pass_specs, + strict=True, ) - ) - batched_video_chunks = [] - batched_audio_chunks = [] - with self._ltx2_model_forward_context(ctx, step): - for index, (model_kwargs_chunk, pass_spec) in enumerate( - zip( - self._split_ltx2_model_kwargs( - batched_model_kwargs, split_sizes - ), - split_pass_specs, - strict=True, + ): + if use_split_pass_kwargs: + self._apply_ltx2_guidance_pass_kwargs( + model_kwargs_chunk, pass_spec ) - ): - if use_split_pass_kwargs: - self._apply_ltx2_guidance_pass_kwargs( - model_kwargs_chunk, pass_spec - ) - else: - model_kwargs_chunk["perturbation_configs"] = ( - split_perturbation_configs[index], - ) - video_chunk, audio_chunk = step.current_model( - **model_kwargs_chunk + else: + model_kwargs_chunk["perturbation_configs"] = ( + split_perturbation_configs[index], ) - batched_video_chunks.append(video_chunk) - batched_audio_chunks.append(audio_chunk) - - batched_video = torch.cat(batched_video_chunks, dim=0) - batched_audio = torch.cat(batched_audio_chunks, dim=0) - else: - perturbation_configs = ( - self._build_ltx2_guidance_perturbation_configs( - execution_pass_specs, batch_size_local - ) - ) - with self._ltx2_model_forward_context(ctx, step): - batched_video, batched_audio = step.current_model( - **batched_model_kwargs, - perturbation_configs=perturbation_configs, + video_chunk, audio_chunk = step.current_model( + **model_kwargs_chunk ) + batched_video_chunks.append(video_chunk) + batched_audio_chunks.append(audio_chunk) - batched_video = batched_video.float() - batched_audio = batched_audio.float() - pass_outputs = { - pass_spec.name: ( - video_chunk, - audio_chunk, + batched_video = torch.cat(batched_video_chunks, dim=0) + batched_audio = torch.cat(batched_audio_chunks, dim=0) + else: + perturbation_configs = ( + self._build_ltx2_guidance_perturbation_configs( + execution_pass_specs, batch_size_local ) - for pass_spec, video_chunk, audio_chunk in zip( - execution_pass_specs, - batched_video.chunk(num_execution_passes, dim=0), - batched_audio.chunk(num_execution_passes, dim=0), - strict=True, + ) + with self._ltx2_model_forward_context(ctx, step): + batched_video, batched_audio = step.current_model( + **batched_model_kwargs, + perturbation_configs=perturbation_configs, ) - } - if not stage1_cfg_parallel: - v_pos, a_v_pos = pass_outputs["cond"] - v_neg, a_v_neg = pass_outputs["neg"] - v_ptb, a_v_ptb = pass_outputs.get("perturbed", (None, None)) - v_mod, a_v_mod = pass_outputs.get("modality", (None, None)) + + batched_video = batched_video.float() + batched_audio = batched_audio.float() + pass_outputs = { + pass_spec.name: ( + video_chunk, + audio_chunk, + ) + for pass_spec, video_chunk, audio_chunk in zip( + execution_pass_specs, + batched_video.chunk(num_execution_passes, dim=0), + batched_audio.chunk(num_execution_passes, dim=0), + strict=True, + ) + } + if not stage1_cfg_parallel: + v_pos, a_v_pos = pass_outputs["cond"] + v_neg, a_v_neg = pass_outputs["neg"] + v_ptb, a_v_ptb = pass_outputs.get("perturbed", (None, None)) + v_mod, a_v_mod = pass_outputs.get("modality", (None, None)) sigma_value_float = float(sigma_value.item()) video_sigma_for_x0: float | torch.Tensor = sigma_value_float diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_cfg_parallel_guidance.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_cfg_parallel_guidance.py new file mode 100644 index 0000000000..adda2a6512 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_cfg_parallel_guidance.py @@ -0,0 +1,108 @@ +import unittest +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2 import ( + denoising as ltx2_denoising, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import ( + LTX2DenoisingStage, +) +from sglang.test.test_utils import CustomTestCase + +_PASS_NAMES = ("cond", "neg", "perturbed", "modality") +_SCALES = dict( + video_cfg_scale=3.0, + video_stg_scale=1.0, + video_rescale_scale=0.7, + video_modality_scale=3.0, + audio_cfg_scale=7.0, + audio_stg_scale=1.0, + audio_rescale_scale=0.7, + audio_modality_scale=3.0, +) + + +class TestLTX2CFGParallelGuidance(CustomTestCase): + """Rank-split LTX-2 multi-guidance must combine exactly like one GPU.""" + + def setUp(self): + gen = torch.Generator().manual_seed(0) + self.sigma = 0.73 + self.video_latents = torch.randn(1, 24, 16, generator=gen).bfloat16() + self.audio_latents = torch.randn(1, 6, 8, generator=gen).bfloat16() + self.video_velocities = { + name: torch.randn(1, 24, 16, generator=gen).bfloat16().float() + for name in _PASS_NAMES + } + self.audio_velocities = { + name: torch.randn(1, 6, 8, generator=gen).bfloat16().float() + for name in _PASS_NAMES + } + + def _single_gpu(self, latents, velocities, prefix): + x0 = { + name: LTX2DenoisingStage._ltx2_velocity_to_x0(latents, v, self.sigma) + for name, v in velocities.items() + } + return LTX2DenoisingStage._ltx2_calculate_guided_x0( + cond=x0["cond"], + uncond_text=x0["neg"], + uncond_perturbed=x0["perturbed"], + uncond_modality=x0["modality"], + cfg_scale=_SCALES[f"{prefix}_cfg_scale"], + stg_scale=_SCALES[f"{prefix}_stg_scale"], + rescale_scale=_SCALES[f"{prefix}_rescale_scale"], + modality_scale=_SCALES[f"{prefix}_modality_scale"], + ) + + def _combine_on_rank(self, rank, world_size, all_reduce): + local = [n for i, n in enumerate(_PASS_NAMES) if i % world_size == rank] + with patch.object(ltx2_denoising, "cfg_model_parallel_all_reduce", all_reduce): + return LTX2DenoisingStage._ltx2_combine_guided_x0_parallel_av( + video_latents=self.video_latents, + audio_latents=self.audio_latents, + local_video_velocities={n: self.video_velocities[n] for n in local}, + local_audio_velocities={n: self.audio_velocities[n] for n in local}, + video_sigma=self.sigma, + audio_sigma=self.sigma, + **_SCALES, + ) + + def test_rank_split_matches_single_gpu_bitwise(self): + expected_video = self._single_gpu( + self.video_latents, self.video_velocities, "video" + ) + expected_audio = self._single_gpu( + self.audio_latents, self.audio_velocities, "audio" + ) + for world_size in (2, 4): + # First pass records each rank's all-reduce contributions; the second + # replays the summed tensors as the collective would return them. + contributions = [[] for _ in range(world_size)] + for rank in range(world_size): + + def record(t, rank=rank): + contributions[rank].append(t.clone()) + return t + + self._combine_on_rank(rank, world_size, record) + + for rank in range(world_size): + calls = iter(range(len(_PASS_NAMES))) + + def summed(t): + i = next(calls) + total = contributions[0][i].clone() + for other in contributions[1:]: + total += other[i] + return total + + video, audio = self._combine_on_rank(rank, world_size, summed) + self.assertTrue(torch.equal(video, expected_video)) + self.assertTrue(torch.equal(audio, expected_audio)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py b/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py index 2d71425ef4..d37dae6c1f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py +++ b/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py @@ -12,14 +12,25 @@ from sglang.multimodal_gen.configs.models.vaes.ltx_video import ( LTXVideoVAEArchConfig, LTXVideoVAEConfig, ) +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.sample.joy_echo import JoyEchoSamplingParams +from sglang.multimodal_gen.configs.sample.sampling_params import ( + DataType, + SamplingParams, +) from sglang.multimodal_gen.runtime.entrypoints.utils import ( _sample_to_uint8_frames, _save_video_ffmpeg, _try_save_video_with_audio, ) from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import AutoencoderKLLTX2Video +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( EncodeWhileDecoding, + can_encode_while_decoding, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.joy_echo.setup import ( + JoyEchoMultishotSetupStage, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.decoding_av import ( LTX2AVDecodingStage, @@ -146,21 +157,33 @@ def test_stream_matches_cpu_postprocessing(tmp_path, dtype, audio): (True, "/tmp/clip.mp4", ["/tmp/clip.mp4"]), ], ) -def test_audio_order_and_encode_fallback(eligible, path, saved): +@pytest.mark.parametrize("initialized,rank", [(False, 0), (True, 0), (True, 1)]) +@pytest.mark.parametrize("enable_memory_bank", [False, True]) +def test_audio_order_and_encode_fallback( + eligible, path, saved, initialized, rank, enable_memory_bank +): stage = object.__new__(LTX2AVDecodingStage) stage.vae = tiny_vae() stage.video_processor = VideoProcessor() calls = [] video = torch.zeros(1, 3, 1, 8, 8) - request = SimpleNamespace( + request = Req( + sampling_params=( + JoyEchoSamplingParams() if enable_memory_bank else SamplingParams() + ), latents=video, - use_diffusion_decoder=False, - return_trajectory_decoded=False, - trajectory_timesteps=None, - trajectory_latents=None, - metrics=None, - rollout_trajectory_data=None, ) + request.use_diffusion_decoder = False + request.return_trajectory_decoded = False + if enable_memory_bank: + setup = JoyEchoMultishotSetupStage( + SimpleNamespace( + multishot_index=0, + _multishot_session_id=None, + reset_memory_bank=mock.Mock(), + ) + ) + setup.forward(request, None) module = LTX2AVDecodingStage.__module__ encoder = mock.Mock() encoder.finish.return_value = saved @@ -177,11 +200,63 @@ def test_audio_order_and_encode_fallback(eligible, path, saved): mock.patch(f"{module}.can_encode_while_decoding", return_value=eligible), mock.patch(f"{module}.streamed_video_output_path", return_value=path), mock.patch(f"{module}.EncodeWhileDecoding", return_value=encoder), + mock.patch(f"{module}.model_parallel_is_initialized", return_value=initialized), + mock.patch( + f"{module}.get_replica_group", + return_value=SimpleNamespace(rank_in_group=rank), + ), + mock.patch.object( + stage.video_processor, + "postprocess_video", + wraps=stage.video_processor.postprocess_video, + ) as postprocess, ): result = stage.forward(request, None) assert calls == (["audio", "video"] if eligible else ["video", "audio"]) if saved is not None: assert result.output is None assert result.output_file_paths == saved + elif initialized and rank != 0 and not enable_memory_bank: + assert result.output is None else: assert result.output.shape == (1, 1, 8, 8, 3) + assert postprocess.call_count == int( + saved is None and (enable_memory_bank or not initialized or rank == 0) + ) + + +@pytest.mark.parametrize("enable_memory_bank", [False, True]) +def test_memory_bank_retains_decoded_frames(enable_memory_bank): + request = Req( + sampling_params=( + JoyEchoSamplingParams() if enable_memory_bank else SamplingParams() + ) + ) + request.save_output = True + request.return_file_paths_only = True + request.return_raw_frames = False + request.data_type = DataType.VIDEO + request.enable_frame_interpolation = False + request.enable_upscaling = False + if enable_memory_bank: + setup = JoyEchoMultishotSetupStage( + SimpleNamespace( + multishot_index=0, + _multishot_session_id=None, + reset_memory_bank=mock.Mock(), + ) + ) + setup.forward(request, None) + else: + with pytest.raises(AttributeError): + request.enable_memory_bank + server_args = SimpleNamespace( + enable_torch_compile=False, pipeline_config=PipelineConfig() + ) + with mock.patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding.current_platform.is_cuda", + return_value=True, + ): + assert can_encode_while_decoding(request, server_args, 1) == ( + not enable_memory_bank + )