[diffusion] fix: skip LTX-2 SP masks only for globally unpadded sequences (#43134)

Co-authored-by: yatholam <yatholam@amd.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-09 15:08:41 +08:00
committed by GitHub
co-authored by yatholam Mick Qian
parent c44a42e6d3
commit 37ae292e6f
5 changed files with 142 additions and 0 deletions
@@ -394,6 +394,7 @@ class LTX2PipelineConfig(PipelineConfig):
# Pad whole frames so `latent_frames` is divisible by `sp_world_size`.
pad_frames = (sp_world_size - (latent_frames % sp_world_size)) % sp_world_size
batch.sp_video_has_padding = pad_frames > 0
if pad_frames:
pad_tokens = int(pad_frames) * int(tokens_per_frame)
pad = torch.zeros(
@@ -443,6 +444,7 @@ class LTX2PipelineConfig(PipelineConfig):
batch.sp_audio_orig_num_frames = int(seq_len)
pad_frames = (sp_world_size - (seq_len % sp_world_size)) % sp_world_size
batch.sp_audio_has_padding = pad_frames > 0
if pad_frames:
pad = torch.zeros(
(audio_latents.shape[0], pad_frames, audio_latents.shape[2]),
@@ -146,6 +146,7 @@ class Req:
did_sp_shard_audio_latents: bool = False
sp_audio_start_frame: int = 0
sp_audio_orig_num_frames: int = 0
sp_audio_has_padding: bool = False
# Audio Parameters
generate_audio: bool = True
@@ -156,6 +157,7 @@ class Req:
raw_latent_shape: torch.Tensor | None = None
did_sp_shard_latents: bool = False
sp_video_start_frame: int = 0
sp_video_has_padding: bool = False
noise_pred: torch.Tensor | list | tuple | None = None
# vae-encoded condition image
image_latent: torch.Tensor | list[torch.Tensor] | None = None
@@ -98,6 +98,7 @@ class JoyEchoDMDDenoisingStage(LTX2AVDenoisingStage):
seq_len=seq_v,
batch_size=batch_size,
key="sp_video_valid_token_count",
has_padding=batch.sp_video_has_padding,
device=model_inputs.latent_model_input.device,
)
video_coords = server_args.pipeline_config.prepare_video_rope_coords_for_sp(
@@ -129,6 +130,7 @@ class JoyEchoDMDDenoisingStage(LTX2AVDenoisingStage):
seq_len=seq_a,
batch_size=batch_size,
key="sp_audio_valid_token_count",
has_padding=batch.sp_audio_has_padding,
device=model_inputs.audio_latent_model_input.device,
)
audio_coords = server_args.pipeline_config.prepare_audio_rope_coords_for_sp(
@@ -9,6 +9,7 @@ from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
is_ltx23_native_variant,
)
from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_sp_world_size,
@@ -967,12 +968,19 @@ class LTX2DenoisingStage(DenoisingStage):
seq_len: int,
batch_size: int,
key: str,
has_padding: bool,
device: torch.device,
) -> torch.Tensor | None:
valid = getattr(batch, key, None)
if valid is None:
return None
valid = int(valid)
# all SP ranks must agree on mask presence to preserve collective order
# a locally all-valid shard still needs a mask when another rank has padding
# eliding an all-valid mask changes attention kernel rounding
# exact requests must keep the reference masked path
if not has_padding and quality_allows(batch.quality, "lossless"):
return None
mask = torch.ones((batch_size, int(seq_len)), device=device, dtype=torch.bool)
if valid < int(seq_len):
mask[:, max(0, valid) :] = False
@@ -1224,6 +1232,7 @@ class LTX2DenoisingStage(DenoisingStage):
seq_len=int(latent_model_input.shape[1]),
batch_size=batch_size,
key="sp_video_valid_token_count",
has_padding=batch.sp_video_has_padding,
device=latent_model_input.device,
)
audio_self_attention_mask = self._build_ltx2_sp_padding_mask(
@@ -1231,6 +1240,7 @@ class LTX2DenoisingStage(DenoisingStage):
seq_len=audio_num_frames_latent,
batch_size=batch_size,
key="sp_audio_valid_token_count",
has_padding=batch.sp_audio_has_padding,
device=audio_latent_model_input.device,
)
a2v_cross_attention_mask = audio_self_attention_mask
@@ -0,0 +1,126 @@
import unittest
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
LTX2DenoisingStage,
)
from sglang.test.test_utils import CustomTestCase
class TestLTX2SPPaddingMask(CustomTestCase):
KEY = "sp_valid_len"
def _build(
self, valid, *, seq_len, batch_size=2, has_padding=True, quality="lossless"
):
batch = SimpleNamespace(
quality=quality, **({self.KEY: valid} if valid is not None else {})
)
return LTX2DenoisingStage._build_ltx2_sp_padding_mask(
batch,
seq_len=seq_len,
batch_size=batch_size,
key=self.KEY,
has_padding=has_padding,
device=torch.device("cpu"),
)
def test_missing_valid_returns_none(self):
# Attribute absent on the batch -> no mask.
self.assertIsNone(self._build(None, seq_len=8))
def test_no_padding_returns_none(self):
# valid == seq_len: an all-True mask would be a no-op, so return None
# to keep the fused (unmasked) attention path.
self.assertIsNone(self._build(8, seq_len=8, has_padding=False))
def test_exact_no_padding_preserves_reference_mask(self):
mask = self._build(8, seq_len=8, has_padding=False, quality="exact")
self.assertTrue(torch.equal(mask, torch.ones(2, 8, dtype=torch.bool)))
def test_high_no_padding_returns_none(self):
self.assertIsNone(self._build(8, seq_len=8, has_padding=False, quality="high"))
def test_valid_greater_than_seq_len_returns_none(self):
# Defensive: valid > seq_len still means no padding.
self.assertIsNone(self._build(12, seq_len=8, has_padding=False))
def test_padding_returns_real_mask(self):
mask = self._build(5, seq_len=8, batch_size=2)
self.assertIsNotNone(mask)
self.assertEqual(mask.shape, (2, 8))
self.assertEqual(mask.dtype, torch.bool)
expected = torch.tensor([True] * 5 + [False] * 3, dtype=torch.bool)
for row in mask:
self.assertTrue(torch.equal(row, expected))
def test_zero_valid_returns_all_false_mask(self):
mask = self._build(0, seq_len=4, batch_size=1)
self.assertIsNotNone(mask)
self.assertEqual(mask.shape, (1, 4))
self.assertFalse(mask.any().item())
def test_sharding_keeps_mask_presence_consistent_across_ranks(self):
config = LTX2PipelineConfig()
target = "sglang.multimodal_gen.configs.pipeline_configs.ltx_2"
for modality in ("video", "audio"):
for world_size in (2, 4):
for frames in (1, 7, 8):
with self.subTest(
modality=modality, world_size=world_size, frames=frames
):
masks = []
for rank in range(world_size):
batch = Req(
height=config.vae_scale_factor * config.patch_size,
width=config.vae_scale_factor * config.patch_size * 2,
)
tokens_per_frame = 2 if modality == "video" else 1
latents = torch.ones(1, frames * tokens_per_frame, 4)
with (
mock.patch(
f"{target}.get_sp_world_size",
return_value=world_size,
),
mock.patch(
f"{target}.get_sp_parallel_rank", return_value=rank
),
):
shard, did_shard = (
config.shard_latents_for_sp(batch, latents)
if modality == "video"
else config.shard_audio_latents_for_sp(
batch, latents
)
)
self.assertTrue(did_shard)
mask = LTX2DenoisingStage._build_ltx2_sp_padding_mask(
batch,
seq_len=shard.shape[1],
batch_size=1,
key=f"sp_{modality}_valid_token_count",
has_padding=vars(batch)[f"sp_{modality}_has_padding"],
device=shard.device,
)
masks.append(mask)
if frames % world_size:
self.assertIsNotNone(mask)
self.assertTrue(
torch.equal(mask, shard[:, :, 0].bool())
)
else:
self.assertIsNone(mask)
if frames % world_size:
gathered = torch.cat(masks, dim=1)
self.assertEqual(
gathered.sum().item(), frames * tokens_per_frame
)
if __name__ == "__main__":
unittest.main()