[diffusion] fix: skip LTX-2 SP masks only for globally unpadded sequences (#43134)
Co-authored-by: yatholam <yatholam@amd.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
yatholam
Mick Qian
parent
c44a42e6d3
commit
37ae292e6f
@@ -394,6 +394,7 @@ class LTX2PipelineConfig(PipelineConfig):
|
||||
|
||||
# Pad whole frames so `latent_frames` is divisible by `sp_world_size`.
|
||||
pad_frames = (sp_world_size - (latent_frames % sp_world_size)) % sp_world_size
|
||||
batch.sp_video_has_padding = pad_frames > 0
|
||||
if pad_frames:
|
||||
pad_tokens = int(pad_frames) * int(tokens_per_frame)
|
||||
pad = torch.zeros(
|
||||
@@ -443,6 +444,7 @@ class LTX2PipelineConfig(PipelineConfig):
|
||||
batch.sp_audio_orig_num_frames = int(seq_len)
|
||||
|
||||
pad_frames = (sp_world_size - (seq_len % sp_world_size)) % sp_world_size
|
||||
batch.sp_audio_has_padding = pad_frames > 0
|
||||
if pad_frames:
|
||||
pad = torch.zeros(
|
||||
(audio_latents.shape[0], pad_frames, audio_latents.shape[2]),
|
||||
|
||||
@@ -146,6 +146,7 @@ class Req:
|
||||
did_sp_shard_audio_latents: bool = False
|
||||
sp_audio_start_frame: int = 0
|
||||
sp_audio_orig_num_frames: int = 0
|
||||
sp_audio_has_padding: bool = False
|
||||
|
||||
# Audio Parameters
|
||||
generate_audio: bool = True
|
||||
@@ -156,6 +157,7 @@ class Req:
|
||||
raw_latent_shape: torch.Tensor | None = None
|
||||
did_sp_shard_latents: bool = False
|
||||
sp_video_start_frame: int = 0
|
||||
sp_video_has_padding: bool = False
|
||||
noise_pred: torch.Tensor | list | tuple | None = None
|
||||
# vae-encoded condition image
|
||||
image_latent: torch.Tensor | list[torch.Tensor] | None = None
|
||||
|
||||
+2
@@ -98,6 +98,7 @@ class JoyEchoDMDDenoisingStage(LTX2AVDenoisingStage):
|
||||
seq_len=seq_v,
|
||||
batch_size=batch_size,
|
||||
key="sp_video_valid_token_count",
|
||||
has_padding=batch.sp_video_has_padding,
|
||||
device=model_inputs.latent_model_input.device,
|
||||
)
|
||||
video_coords = server_args.pipeline_config.prepare_video_rope_coords_for_sp(
|
||||
@@ -129,6 +130,7 @@ class JoyEchoDMDDenoisingStage(LTX2AVDenoisingStage):
|
||||
seq_len=seq_a,
|
||||
batch_size=batch_size,
|
||||
key="sp_audio_valid_token_count",
|
||||
has_padding=batch.sp_audio_has_padding,
|
||||
device=model_inputs.audio_latent_model_input.device,
|
||||
)
|
||||
audio_coords = server_args.pipeline_config.prepare_audio_rope_coords_for_sp(
|
||||
|
||||
+10
@@ -9,6 +9,7 @@ from diffusers.utils.torch_utils import randn_tensor
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
|
||||
is_ltx23_native_variant,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import quality_allows
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_sp_world_size,
|
||||
@@ -967,12 +968,19 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
seq_len: int,
|
||||
batch_size: int,
|
||||
key: str,
|
||||
has_padding: bool,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
valid = getattr(batch, key, None)
|
||||
if valid is None:
|
||||
return None
|
||||
valid = int(valid)
|
||||
# all SP ranks must agree on mask presence to preserve collective order
|
||||
# a locally all-valid shard still needs a mask when another rank has padding
|
||||
# eliding an all-valid mask changes attention kernel rounding
|
||||
# exact requests must keep the reference masked path
|
||||
if not has_padding and quality_allows(batch.quality, "lossless"):
|
||||
return None
|
||||
mask = torch.ones((batch_size, int(seq_len)), device=device, dtype=torch.bool)
|
||||
if valid < int(seq_len):
|
||||
mask[:, max(0, valid) :] = False
|
||||
@@ -1224,6 +1232,7 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
seq_len=int(latent_model_input.shape[1]),
|
||||
batch_size=batch_size,
|
||||
key="sp_video_valid_token_count",
|
||||
has_padding=batch.sp_video_has_padding,
|
||||
device=latent_model_input.device,
|
||||
)
|
||||
audio_self_attention_mask = self._build_ltx2_sp_padding_mask(
|
||||
@@ -1231,6 +1240,7 @@ class LTX2DenoisingStage(DenoisingStage):
|
||||
seq_len=audio_num_frames_latent,
|
||||
batch_size=batch_size,
|
||||
key="sp_audio_valid_token_count",
|
||||
has_padding=batch.sp_audio_has_padding,
|
||||
device=audio_latent_model_input.device,
|
||||
)
|
||||
a2v_cross_attention_mask = audio_self_attention_mask
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
|
||||
LTX2DenoisingStage,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class TestLTX2SPPaddingMask(CustomTestCase):
|
||||
KEY = "sp_valid_len"
|
||||
|
||||
def _build(
|
||||
self, valid, *, seq_len, batch_size=2, has_padding=True, quality="lossless"
|
||||
):
|
||||
batch = SimpleNamespace(
|
||||
quality=quality, **({self.KEY: valid} if valid is not None else {})
|
||||
)
|
||||
return LTX2DenoisingStage._build_ltx2_sp_padding_mask(
|
||||
batch,
|
||||
seq_len=seq_len,
|
||||
batch_size=batch_size,
|
||||
key=self.KEY,
|
||||
has_padding=has_padding,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
def test_missing_valid_returns_none(self):
|
||||
# Attribute absent on the batch -> no mask.
|
||||
self.assertIsNone(self._build(None, seq_len=8))
|
||||
|
||||
def test_no_padding_returns_none(self):
|
||||
# valid == seq_len: an all-True mask would be a no-op, so return None
|
||||
# to keep the fused (unmasked) attention path.
|
||||
self.assertIsNone(self._build(8, seq_len=8, has_padding=False))
|
||||
|
||||
def test_exact_no_padding_preserves_reference_mask(self):
|
||||
mask = self._build(8, seq_len=8, has_padding=False, quality="exact")
|
||||
self.assertTrue(torch.equal(mask, torch.ones(2, 8, dtype=torch.bool)))
|
||||
|
||||
def test_high_no_padding_returns_none(self):
|
||||
self.assertIsNone(self._build(8, seq_len=8, has_padding=False, quality="high"))
|
||||
|
||||
def test_valid_greater_than_seq_len_returns_none(self):
|
||||
# Defensive: valid > seq_len still means no padding.
|
||||
self.assertIsNone(self._build(12, seq_len=8, has_padding=False))
|
||||
|
||||
def test_padding_returns_real_mask(self):
|
||||
mask = self._build(5, seq_len=8, batch_size=2)
|
||||
self.assertIsNotNone(mask)
|
||||
self.assertEqual(mask.shape, (2, 8))
|
||||
self.assertEqual(mask.dtype, torch.bool)
|
||||
expected = torch.tensor([True] * 5 + [False] * 3, dtype=torch.bool)
|
||||
for row in mask:
|
||||
self.assertTrue(torch.equal(row, expected))
|
||||
|
||||
def test_zero_valid_returns_all_false_mask(self):
|
||||
mask = self._build(0, seq_len=4, batch_size=1)
|
||||
self.assertIsNotNone(mask)
|
||||
self.assertEqual(mask.shape, (1, 4))
|
||||
self.assertFalse(mask.any().item())
|
||||
|
||||
def test_sharding_keeps_mask_presence_consistent_across_ranks(self):
|
||||
config = LTX2PipelineConfig()
|
||||
target = "sglang.multimodal_gen.configs.pipeline_configs.ltx_2"
|
||||
for modality in ("video", "audio"):
|
||||
for world_size in (2, 4):
|
||||
for frames in (1, 7, 8):
|
||||
with self.subTest(
|
||||
modality=modality, world_size=world_size, frames=frames
|
||||
):
|
||||
masks = []
|
||||
for rank in range(world_size):
|
||||
batch = Req(
|
||||
height=config.vae_scale_factor * config.patch_size,
|
||||
width=config.vae_scale_factor * config.patch_size * 2,
|
||||
)
|
||||
tokens_per_frame = 2 if modality == "video" else 1
|
||||
latents = torch.ones(1, frames * tokens_per_frame, 4)
|
||||
with (
|
||||
mock.patch(
|
||||
f"{target}.get_sp_world_size",
|
||||
return_value=world_size,
|
||||
),
|
||||
mock.patch(
|
||||
f"{target}.get_sp_parallel_rank", return_value=rank
|
||||
),
|
||||
):
|
||||
shard, did_shard = (
|
||||
config.shard_latents_for_sp(batch, latents)
|
||||
if modality == "video"
|
||||
else config.shard_audio_latents_for_sp(
|
||||
batch, latents
|
||||
)
|
||||
)
|
||||
self.assertTrue(did_shard)
|
||||
mask = LTX2DenoisingStage._build_ltx2_sp_padding_mask(
|
||||
batch,
|
||||
seq_len=shard.shape[1],
|
||||
batch_size=1,
|
||||
key=f"sp_{modality}_valid_token_count",
|
||||
has_padding=vars(batch)[f"sp_{modality}_has_padding"],
|
||||
device=shard.device,
|
||||
)
|
||||
masks.append(mask)
|
||||
if frames % world_size:
|
||||
self.assertIsNotNone(mask)
|
||||
self.assertTrue(
|
||||
torch.equal(mask, shard[:, :, 0].bool())
|
||||
)
|
||||
else:
|
||||
self.assertIsNone(mask)
|
||||
if frames % world_size:
|
||||
gathered = torch.cat(masks, dim=1)
|
||||
self.assertEqual(
|
||||
gathered.sum().item(), frames * tokens_per_frame
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user