[diffusion] perf: split and batch LTX-2.3 one-stage guidance passes under CFG parallel (#43684)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -33,6 +33,7 @@ def can_encode_while_decoding(
|
||||
batch.save_output
|
||||
and batch.return_file_paths_only
|
||||
and not batch.return_raw_frames
|
||||
and not (batch.extra or {}).get("retain_decoded_frames", False)
|
||||
and batch.data_type == DataType.VIDEO
|
||||
and not batch.enable_frame_interpolation
|
||||
and not batch.enable_upscaling
|
||||
|
||||
+1
@@ -48,6 +48,7 @@ class JoyEchoMultishotSetupStage(PipelineStage):
|
||||
logger.info("JoyEcho memory bank reset for new multi-shot session")
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
batch.extra["retain_decoded_frames"] = batch.enable_memory_bank
|
||||
if not batch.enable_memory_bank:
|
||||
return batch
|
||||
|
||||
|
||||
+10
-2
@@ -3,7 +3,11 @@ from collections.abc import Callable
|
||||
import torch
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_replica_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
@@ -152,7 +156,11 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
raise
|
||||
if output_paths is not None:
|
||||
output_batch.output_file_paths = output_paths
|
||||
else:
|
||||
elif (
|
||||
batch.extra.get("retain_decoded_frames", False)
|
||||
or not model_parallel_is_initialized()
|
||||
or get_replica_group().rank_in_group == 0
|
||||
):
|
||||
output_batch.output = self.video_processor.postprocess_video(
|
||||
video, output_type="np"
|
||||
)
|
||||
|
||||
+154
-220
@@ -1947,20 +1947,28 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
float(stage1_guider_params["video_modality_scale"]) != 1.0
|
||||
or float(stage1_guider_params["audio_modality_scale"]) != 1.0
|
||||
)
|
||||
stage1_cfg_parallel = (
|
||||
server_args.enable_cfg_parallel and not ctx.use_ltx23_legacy_one_stage
|
||||
)
|
||||
# Legacy LTX-2.3 one-stage runs the same guidance passes, so CFG
|
||||
# parallel splits them across ranks like the other LTX-2 paths.
|
||||
stage1_cfg_parallel = server_args.enable_cfg_parallel
|
||||
stage1_cfg_rank = (
|
||||
get_classifier_free_guidance_rank() if stage1_cfg_parallel else 0
|
||||
)
|
||||
stage1_cfg_world_size = (
|
||||
get_classifier_free_guidance_world_size() if stage1_cfg_parallel else 1
|
||||
)
|
||||
# Legacy LTX-2.3 one-stage batches its guidance passes into one forward
|
||||
# (perturbation masks are exact per item; only GEMM/attention rounding
|
||||
# can move with the batch size). quality="exact" keeps the reference
|
||||
# one-pass-per-forward schedule, which is bit-identical to it.
|
||||
use_legacy_exact_passes = ctx.use_ltx23_legacy_one_stage and not (
|
||||
quality_allows(batch.quality, "lossless")
|
||||
)
|
||||
# NOTE: this flag must be identical across all SP ranks so that every
|
||||
# rank executes the same number of model-forward calls (each of which
|
||||
# contains NCCL collectives).
|
||||
use_split_stage1_guided_passes = (
|
||||
server_args.pipeline_class_name == "LTX2TwoStageHQPipeline"
|
||||
or use_legacy_exact_passes
|
||||
or (
|
||||
is_ltx2_two_stage_pipeline_name(server_args.pipeline_class_name)
|
||||
and int(getattr(batch, "ltx2_num_image_tokens", 0)) > 0
|
||||
@@ -1980,6 +1988,7 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
# perturbation_configs: one config dict per expanded item.
|
||||
use_split_pass_kwargs = (
|
||||
server_args.pipeline_class_name == "LTX2TwoStageHQPipeline"
|
||||
or use_legacy_exact_passes
|
||||
)
|
||||
skip_v2a_cross_attn_for_video_gt = bool(
|
||||
batch.extra.get("ltx2_skip_v2a_cross_attn_for_video_gt", False)
|
||||
@@ -2005,240 +2014,165 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
ctx, batch, model_inputs_local
|
||||
)
|
||||
|
||||
if ctx.use_ltx23_legacy_one_stage:
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
v_pos, a_v_pos = step.current_model(
|
||||
**self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
base_model_kwargs_local,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
disable_v2a_cross_attn=(
|
||||
skip_v2a_cross_attn_for_video_gt
|
||||
),
|
||||
)
|
||||
)
|
||||
v_neg, a_v_neg = step.current_model(
|
||||
**self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
base_model_kwargs_local,
|
||||
encoder_hidden_states=negative_encoder_hidden_states,
|
||||
audio_encoder_hidden_states=negative_audio_encoder_hidden_states,
|
||||
encoder_attention_mask=negative_encoder_attention_mask,
|
||||
disable_v2a_cross_attn=(
|
||||
skip_v2a_cross_attn_for_video_gt
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
v_pos = v_pos.float()
|
||||
a_v_pos = a_v_pos.float()
|
||||
v_neg = v_neg.float()
|
||||
a_v_neg = a_v_neg.float()
|
||||
|
||||
v_ptb = None
|
||||
a_v_ptb = None
|
||||
if need_perturbed:
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
v_ptb, a_v_ptb = step.current_model(
|
||||
**self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
base_model_kwargs_local,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
skip_video_self_attn_blocks=tuple(
|
||||
stage1_guider_params["video_stg_blocks"]
|
||||
),
|
||||
skip_audio_self_attn_blocks=tuple(
|
||||
stage1_guider_params["audio_stg_blocks"]
|
||||
),
|
||||
disable_v2a_cross_attn=(
|
||||
skip_v2a_cross_attn_for_video_gt
|
||||
),
|
||||
)
|
||||
)
|
||||
v_ptb = v_ptb.float()
|
||||
a_v_ptb = a_v_ptb.float()
|
||||
|
||||
v_mod = None
|
||||
a_v_mod = None
|
||||
if need_modality:
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
v_mod, a_v_mod = step.current_model(
|
||||
**self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
base_model_kwargs_local,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
disable_a2v_cross_attn=True,
|
||||
disable_v2a_cross_attn=True,
|
||||
)
|
||||
)
|
||||
v_mod = v_mod.float()
|
||||
a_v_mod = a_v_mod.float()
|
||||
else:
|
||||
pass_specs: list[LTX2GuidancePassSpec] = [
|
||||
pass_specs: list[LTX2GuidancePassSpec] = [
|
||||
LTX2GuidancePassSpec(
|
||||
name="cond",
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt,
|
||||
),
|
||||
LTX2GuidancePassSpec(
|
||||
name="neg",
|
||||
encoder_hidden_states=negative_encoder_hidden_states,
|
||||
audio_encoder_hidden_states=negative_audio_encoder_hidden_states,
|
||||
encoder_attention_mask=negative_encoder_attention_mask,
|
||||
disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt,
|
||||
),
|
||||
]
|
||||
if need_perturbed:
|
||||
pass_specs.append(
|
||||
LTX2GuidancePassSpec(
|
||||
name="cond",
|
||||
name="perturbed",
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
skip_video_self_attn_blocks=tuple(
|
||||
stage1_guider_params["video_stg_blocks"]
|
||||
),
|
||||
skip_audio_self_attn_blocks=tuple(
|
||||
stage1_guider_params["audio_stg_blocks"]
|
||||
),
|
||||
disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt,
|
||||
),
|
||||
)
|
||||
)
|
||||
if need_modality:
|
||||
pass_specs.append(
|
||||
LTX2GuidancePassSpec(
|
||||
name="neg",
|
||||
encoder_hidden_states=negative_encoder_hidden_states,
|
||||
audio_encoder_hidden_states=negative_audio_encoder_hidden_states,
|
||||
encoder_attention_mask=negative_encoder_attention_mask,
|
||||
disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt,
|
||||
),
|
||||
name="modality",
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
disable_a2v_cross_attn=True,
|
||||
disable_v2a_cross_attn=True,
|
||||
)
|
||||
)
|
||||
|
||||
execution_pass_specs = (
|
||||
[
|
||||
pass_spec
|
||||
for index, pass_spec in enumerate(pass_specs)
|
||||
if index % stage1_cfg_world_size == stage1_cfg_rank
|
||||
]
|
||||
if need_perturbed:
|
||||
pass_specs.append(
|
||||
LTX2GuidancePassSpec(
|
||||
name="perturbed",
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
skip_video_self_attn_blocks=tuple(
|
||||
stage1_guider_params["video_stg_blocks"]
|
||||
),
|
||||
skip_audio_self_attn_blocks=tuple(
|
||||
stage1_guider_params["audio_stg_blocks"]
|
||||
),
|
||||
disable_v2a_cross_attn=skip_v2a_cross_attn_for_video_gt,
|
||||
)
|
||||
)
|
||||
if need_modality:
|
||||
pass_specs.append(
|
||||
LTX2GuidancePassSpec(
|
||||
name="modality",
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
audio_encoder_hidden_states=audio_encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
disable_a2v_cross_attn=True,
|
||||
disable_v2a_cross_attn=True,
|
||||
)
|
||||
)
|
||||
|
||||
execution_pass_specs = (
|
||||
if stage1_cfg_parallel
|
||||
else pass_specs
|
||||
)
|
||||
num_execution_passes = len(execution_pass_specs)
|
||||
if num_execution_passes == 0:
|
||||
raise ValueError(
|
||||
"LTX2 stage-1 CFG parallel degree exceeds guidance pass count."
|
||||
)
|
||||
expanded_batch_size = batch_size_local * num_execution_passes
|
||||
batched_model_kwargs = self._repeat_ltx2_model_kwargs_batch(
|
||||
base_model_kwargs_local, expanded_batch_size
|
||||
)
|
||||
batched_model_kwargs = self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
batched_model_kwargs,
|
||||
encoder_hidden_states=torch.cat(
|
||||
[
|
||||
pass_spec
|
||||
for index, pass_spec in enumerate(pass_specs)
|
||||
if index % stage1_cfg_world_size == stage1_cfg_rank
|
||||
]
|
||||
if stage1_cfg_parallel
|
||||
else pass_specs
|
||||
)
|
||||
num_execution_passes = len(execution_pass_specs)
|
||||
if num_execution_passes == 0:
|
||||
raise ValueError(
|
||||
"LTX2 stage-1 CFG parallel degree exceeds guidance pass count."
|
||||
)
|
||||
expanded_batch_size = batch_size_local * num_execution_passes
|
||||
batched_model_kwargs = self._repeat_ltx2_model_kwargs_batch(
|
||||
base_model_kwargs_local, expanded_batch_size
|
||||
)
|
||||
batched_model_kwargs = self._build_ltx2_model_kwargs(
|
||||
ctx,
|
||||
batched_model_kwargs,
|
||||
encoder_hidden_states=torch.cat(
|
||||
[
|
||||
pass_spec.encoder_hidden_states
|
||||
for pass_spec in execution_pass_specs
|
||||
],
|
||||
dim=0,
|
||||
),
|
||||
audio_encoder_hidden_states=torch.cat(
|
||||
[
|
||||
pass_spec.audio_encoder_hidden_states
|
||||
for pass_spec in execution_pass_specs
|
||||
],
|
||||
dim=0,
|
||||
),
|
||||
encoder_attention_mask=self._cat_or_none(
|
||||
[
|
||||
pass_spec.encoder_attention_mask
|
||||
for pass_spec in execution_pass_specs
|
||||
]
|
||||
),
|
||||
)
|
||||
if use_split_stage1_guided_passes:
|
||||
split_sizes = [1] * expanded_batch_size
|
||||
split_pass_specs = tuple(
|
||||
pass_spec
|
||||
pass_spec.encoder_hidden_states
|
||||
for pass_spec in execution_pass_specs
|
||||
for _ in range(batch_size_local)
|
||||
],
|
||||
dim=0,
|
||||
),
|
||||
audio_encoder_hidden_states=torch.cat(
|
||||
[
|
||||
pass_spec.audio_encoder_hidden_states
|
||||
for pass_spec in execution_pass_specs
|
||||
],
|
||||
dim=0,
|
||||
),
|
||||
encoder_attention_mask=self._cat_or_none(
|
||||
[
|
||||
pass_spec.encoder_attention_mask
|
||||
for pass_spec in execution_pass_specs
|
||||
]
|
||||
),
|
||||
)
|
||||
if use_split_stage1_guided_passes:
|
||||
split_sizes = [1] * expanded_batch_size
|
||||
split_pass_specs = tuple(
|
||||
pass_spec
|
||||
for pass_spec in execution_pass_specs
|
||||
for _ in range(batch_size_local)
|
||||
)
|
||||
split_perturbation_configs = (
|
||||
()
|
||||
if use_split_pass_kwargs
|
||||
else self._build_ltx2_guidance_perturbation_configs(
|
||||
execution_pass_specs, batch_size_local
|
||||
)
|
||||
split_perturbation_configs = (
|
||||
()
|
||||
if use_split_pass_kwargs
|
||||
else self._build_ltx2_guidance_perturbation_configs(
|
||||
execution_pass_specs, batch_size_local
|
||||
)
|
||||
batched_video_chunks = []
|
||||
batched_audio_chunks = []
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
for index, (model_kwargs_chunk, pass_spec) in enumerate(
|
||||
zip(
|
||||
self._split_ltx2_model_kwargs(
|
||||
batched_model_kwargs, split_sizes
|
||||
),
|
||||
split_pass_specs,
|
||||
strict=True,
|
||||
)
|
||||
)
|
||||
batched_video_chunks = []
|
||||
batched_audio_chunks = []
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
for index, (model_kwargs_chunk, pass_spec) in enumerate(
|
||||
zip(
|
||||
self._split_ltx2_model_kwargs(
|
||||
batched_model_kwargs, split_sizes
|
||||
),
|
||||
split_pass_specs,
|
||||
strict=True,
|
||||
):
|
||||
if use_split_pass_kwargs:
|
||||
self._apply_ltx2_guidance_pass_kwargs(
|
||||
model_kwargs_chunk, pass_spec
|
||||
)
|
||||
):
|
||||
if use_split_pass_kwargs:
|
||||
self._apply_ltx2_guidance_pass_kwargs(
|
||||
model_kwargs_chunk, pass_spec
|
||||
)
|
||||
else:
|
||||
model_kwargs_chunk["perturbation_configs"] = (
|
||||
split_perturbation_configs[index],
|
||||
)
|
||||
video_chunk, audio_chunk = step.current_model(
|
||||
**model_kwargs_chunk
|
||||
else:
|
||||
model_kwargs_chunk["perturbation_configs"] = (
|
||||
split_perturbation_configs[index],
|
||||
)
|
||||
batched_video_chunks.append(video_chunk)
|
||||
batched_audio_chunks.append(audio_chunk)
|
||||
|
||||
batched_video = torch.cat(batched_video_chunks, dim=0)
|
||||
batched_audio = torch.cat(batched_audio_chunks, dim=0)
|
||||
else:
|
||||
perturbation_configs = (
|
||||
self._build_ltx2_guidance_perturbation_configs(
|
||||
execution_pass_specs, batch_size_local
|
||||
)
|
||||
)
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
batched_video, batched_audio = step.current_model(
|
||||
**batched_model_kwargs,
|
||||
perturbation_configs=perturbation_configs,
|
||||
video_chunk, audio_chunk = step.current_model(
|
||||
**model_kwargs_chunk
|
||||
)
|
||||
batched_video_chunks.append(video_chunk)
|
||||
batched_audio_chunks.append(audio_chunk)
|
||||
|
||||
batched_video = batched_video.float()
|
||||
batched_audio = batched_audio.float()
|
||||
pass_outputs = {
|
||||
pass_spec.name: (
|
||||
video_chunk,
|
||||
audio_chunk,
|
||||
batched_video = torch.cat(batched_video_chunks, dim=0)
|
||||
batched_audio = torch.cat(batched_audio_chunks, dim=0)
|
||||
else:
|
||||
perturbation_configs = (
|
||||
self._build_ltx2_guidance_perturbation_configs(
|
||||
execution_pass_specs, batch_size_local
|
||||
)
|
||||
for pass_spec, video_chunk, audio_chunk in zip(
|
||||
execution_pass_specs,
|
||||
batched_video.chunk(num_execution_passes, dim=0),
|
||||
batched_audio.chunk(num_execution_passes, dim=0),
|
||||
strict=True,
|
||||
)
|
||||
with self._ltx2_model_forward_context(ctx, step):
|
||||
batched_video, batched_audio = step.current_model(
|
||||
**batched_model_kwargs,
|
||||
perturbation_configs=perturbation_configs,
|
||||
)
|
||||
}
|
||||
if not stage1_cfg_parallel:
|
||||
v_pos, a_v_pos = pass_outputs["cond"]
|
||||
v_neg, a_v_neg = pass_outputs["neg"]
|
||||
v_ptb, a_v_ptb = pass_outputs.get("perturbed", (None, None))
|
||||
v_mod, a_v_mod = pass_outputs.get("modality", (None, None))
|
||||
|
||||
batched_video = batched_video.float()
|
||||
batched_audio = batched_audio.float()
|
||||
pass_outputs = {
|
||||
pass_spec.name: (
|
||||
video_chunk,
|
||||
audio_chunk,
|
||||
)
|
||||
for pass_spec, video_chunk, audio_chunk in zip(
|
||||
execution_pass_specs,
|
||||
batched_video.chunk(num_execution_passes, dim=0),
|
||||
batched_audio.chunk(num_execution_passes, dim=0),
|
||||
strict=True,
|
||||
)
|
||||
}
|
||||
if not stage1_cfg_parallel:
|
||||
v_pos, a_v_pos = pass_outputs["cond"]
|
||||
v_neg, a_v_neg = pass_outputs["neg"]
|
||||
v_ptb, a_v_ptb = pass_outputs.get("perturbed", (None, None))
|
||||
v_mod, a_v_mod = pass_outputs.get("modality", (None, None))
|
||||
|
||||
sigma_value_float = float(sigma_value.item())
|
||||
video_sigma_for_x0: float | torch.Tensor = sigma_value_float
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2 import (
|
||||
denoising as ltx2_denoising,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
|
||||
LTX2DenoisingStage,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
_PASS_NAMES = ("cond", "neg", "perturbed", "modality")
|
||||
_SCALES = dict(
|
||||
video_cfg_scale=3.0,
|
||||
video_stg_scale=1.0,
|
||||
video_rescale_scale=0.7,
|
||||
video_modality_scale=3.0,
|
||||
audio_cfg_scale=7.0,
|
||||
audio_stg_scale=1.0,
|
||||
audio_rescale_scale=0.7,
|
||||
audio_modality_scale=3.0,
|
||||
)
|
||||
|
||||
|
||||
class TestLTX2CFGParallelGuidance(CustomTestCase):
|
||||
"""Rank-split LTX-2 multi-guidance must combine exactly like one GPU."""
|
||||
|
||||
def setUp(self):
|
||||
gen = torch.Generator().manual_seed(0)
|
||||
self.sigma = 0.73
|
||||
self.video_latents = torch.randn(1, 24, 16, generator=gen).bfloat16()
|
||||
self.audio_latents = torch.randn(1, 6, 8, generator=gen).bfloat16()
|
||||
self.video_velocities = {
|
||||
name: torch.randn(1, 24, 16, generator=gen).bfloat16().float()
|
||||
for name in _PASS_NAMES
|
||||
}
|
||||
self.audio_velocities = {
|
||||
name: torch.randn(1, 6, 8, generator=gen).bfloat16().float()
|
||||
for name in _PASS_NAMES
|
||||
}
|
||||
|
||||
def _single_gpu(self, latents, velocities, prefix):
|
||||
x0 = {
|
||||
name: LTX2DenoisingStage._ltx2_velocity_to_x0(latents, v, self.sigma)
|
||||
for name, v in velocities.items()
|
||||
}
|
||||
return LTX2DenoisingStage._ltx2_calculate_guided_x0(
|
||||
cond=x0["cond"],
|
||||
uncond_text=x0["neg"],
|
||||
uncond_perturbed=x0["perturbed"],
|
||||
uncond_modality=x0["modality"],
|
||||
cfg_scale=_SCALES[f"{prefix}_cfg_scale"],
|
||||
stg_scale=_SCALES[f"{prefix}_stg_scale"],
|
||||
rescale_scale=_SCALES[f"{prefix}_rescale_scale"],
|
||||
modality_scale=_SCALES[f"{prefix}_modality_scale"],
|
||||
)
|
||||
|
||||
def _combine_on_rank(self, rank, world_size, all_reduce):
|
||||
local = [n for i, n in enumerate(_PASS_NAMES) if i % world_size == rank]
|
||||
with patch.object(ltx2_denoising, "cfg_model_parallel_all_reduce", all_reduce):
|
||||
return LTX2DenoisingStage._ltx2_combine_guided_x0_parallel_av(
|
||||
video_latents=self.video_latents,
|
||||
audio_latents=self.audio_latents,
|
||||
local_video_velocities={n: self.video_velocities[n] for n in local},
|
||||
local_audio_velocities={n: self.audio_velocities[n] for n in local},
|
||||
video_sigma=self.sigma,
|
||||
audio_sigma=self.sigma,
|
||||
**_SCALES,
|
||||
)
|
||||
|
||||
def test_rank_split_matches_single_gpu_bitwise(self):
|
||||
expected_video = self._single_gpu(
|
||||
self.video_latents, self.video_velocities, "video"
|
||||
)
|
||||
expected_audio = self._single_gpu(
|
||||
self.audio_latents, self.audio_velocities, "audio"
|
||||
)
|
||||
for world_size in (2, 4):
|
||||
# First pass records each rank's all-reduce contributions; the second
|
||||
# replays the summed tensors as the collective would return them.
|
||||
contributions = [[] for _ in range(world_size)]
|
||||
for rank in range(world_size):
|
||||
|
||||
def record(t, rank=rank):
|
||||
contributions[rank].append(t.clone())
|
||||
return t
|
||||
|
||||
self._combine_on_rank(rank, world_size, record)
|
||||
|
||||
for rank in range(world_size):
|
||||
calls = iter(range(len(_PASS_NAMES)))
|
||||
|
||||
def summed(t):
|
||||
i = next(calls)
|
||||
total = contributions[0][i].clone()
|
||||
for other in contributions[1:]:
|
||||
total += other[i]
|
||||
return total
|
||||
|
||||
video, audio = self._combine_on_rank(rank, world_size, summed)
|
||||
self.assertTrue(torch.equal(video, expected_video))
|
||||
self.assertTrue(torch.equal(audio, expected_audio))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,14 +12,25 @@ from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
|
||||
LTXVideoVAEArchConfig,
|
||||
LTXVideoVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.joy_echo import JoyEchoSamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
_sample_to_uint8_frames,
|
||||
_save_video_ffmpeg,
|
||||
_try_save_video_with_audio,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import AutoencoderKLLTX2Video
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
can_encode_while_decoding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.joy_echo.setup import (
|
||||
JoyEchoMultishotSetupStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.decoding_av import (
|
||||
LTX2AVDecodingStage,
|
||||
@@ -146,21 +157,33 @@ def test_stream_matches_cpu_postprocessing(tmp_path, dtype, audio):
|
||||
(True, "/tmp/clip.mp4", ["/tmp/clip.mp4"]),
|
||||
],
|
||||
)
|
||||
def test_audio_order_and_encode_fallback(eligible, path, saved):
|
||||
@pytest.mark.parametrize("initialized,rank", [(False, 0), (True, 0), (True, 1)])
|
||||
@pytest.mark.parametrize("enable_memory_bank", [False, True])
|
||||
def test_audio_order_and_encode_fallback(
|
||||
eligible, path, saved, initialized, rank, enable_memory_bank
|
||||
):
|
||||
stage = object.__new__(LTX2AVDecodingStage)
|
||||
stage.vae = tiny_vae()
|
||||
stage.video_processor = VideoProcessor()
|
||||
calls = []
|
||||
video = torch.zeros(1, 3, 1, 8, 8)
|
||||
request = SimpleNamespace(
|
||||
request = Req(
|
||||
sampling_params=(
|
||||
JoyEchoSamplingParams() if enable_memory_bank else SamplingParams()
|
||||
),
|
||||
latents=video,
|
||||
use_diffusion_decoder=False,
|
||||
return_trajectory_decoded=False,
|
||||
trajectory_timesteps=None,
|
||||
trajectory_latents=None,
|
||||
metrics=None,
|
||||
rollout_trajectory_data=None,
|
||||
)
|
||||
request.use_diffusion_decoder = False
|
||||
request.return_trajectory_decoded = False
|
||||
if enable_memory_bank:
|
||||
setup = JoyEchoMultishotSetupStage(
|
||||
SimpleNamespace(
|
||||
multishot_index=0,
|
||||
_multishot_session_id=None,
|
||||
reset_memory_bank=mock.Mock(),
|
||||
)
|
||||
)
|
||||
setup.forward(request, None)
|
||||
module = LTX2AVDecodingStage.__module__
|
||||
encoder = mock.Mock()
|
||||
encoder.finish.return_value = saved
|
||||
@@ -177,11 +200,63 @@ def test_audio_order_and_encode_fallback(eligible, path, saved):
|
||||
mock.patch(f"{module}.can_encode_while_decoding", return_value=eligible),
|
||||
mock.patch(f"{module}.streamed_video_output_path", return_value=path),
|
||||
mock.patch(f"{module}.EncodeWhileDecoding", return_value=encoder),
|
||||
mock.patch(f"{module}.model_parallel_is_initialized", return_value=initialized),
|
||||
mock.patch(
|
||||
f"{module}.get_replica_group",
|
||||
return_value=SimpleNamespace(rank_in_group=rank),
|
||||
),
|
||||
mock.patch.object(
|
||||
stage.video_processor,
|
||||
"postprocess_video",
|
||||
wraps=stage.video_processor.postprocess_video,
|
||||
) as postprocess,
|
||||
):
|
||||
result = stage.forward(request, None)
|
||||
assert calls == (["audio", "video"] if eligible else ["video", "audio"])
|
||||
if saved is not None:
|
||||
assert result.output is None
|
||||
assert result.output_file_paths == saved
|
||||
elif initialized and rank != 0 and not enable_memory_bank:
|
||||
assert result.output is None
|
||||
else:
|
||||
assert result.output.shape == (1, 1, 8, 8, 3)
|
||||
assert postprocess.call_count == int(
|
||||
saved is None and (enable_memory_bank or not initialized or rank == 0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_memory_bank", [False, True])
|
||||
def test_memory_bank_retains_decoded_frames(enable_memory_bank):
|
||||
request = Req(
|
||||
sampling_params=(
|
||||
JoyEchoSamplingParams() if enable_memory_bank else SamplingParams()
|
||||
)
|
||||
)
|
||||
request.save_output = True
|
||||
request.return_file_paths_only = True
|
||||
request.return_raw_frames = False
|
||||
request.data_type = DataType.VIDEO
|
||||
request.enable_frame_interpolation = False
|
||||
request.enable_upscaling = False
|
||||
if enable_memory_bank:
|
||||
setup = JoyEchoMultishotSetupStage(
|
||||
SimpleNamespace(
|
||||
multishot_index=0,
|
||||
_multishot_session_id=None,
|
||||
reset_memory_bank=mock.Mock(),
|
||||
)
|
||||
)
|
||||
setup.forward(request, None)
|
||||
else:
|
||||
with pytest.raises(AttributeError):
|
||||
request.enable_memory_bank
|
||||
server_args = SimpleNamespace(
|
||||
enable_torch_compile=False, pipeline_config=PipelineConfig()
|
||||
)
|
||||
with mock.patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding.current_platform.is_cuda",
|
||||
return_value=True,
|
||||
):
|
||||
assert can_encode_while_decoding(request, server_args, 1) == (
|
||||
not enable_memory_bank
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user