[Fix] diffusion: Offset LTX-2.3 BCG video RoPE by the SP shard start (#45995)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-23 21:14:37 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 9b46f27090
commit 2fbd094a8c
2 changed files with 57 additions and 0 deletions
@@ -114,6 +114,7 @@ def _prepare_ltx2_rope_coords_for_bcg(
video_coords: torch.Tensor | None,
audio_coords: torch.Tensor | None,
num_frames: int,
video_start_frame: int,
height: int,
width: int,
audio_num_frames: int,
@@ -137,6 +138,7 @@ def _prepare_ltx2_rope_coords_for_bcg(
width=width,
device=latent_model_input.device,
fps=fps,
start_frame=video_start_frame,
)
if audio_coords is None:
audio_coords = current_model.audio_rope.prepare_audio_coords(
@@ -1120,6 +1122,7 @@ class LTX2DenoisingStage(DenoisingStage):
video_coords=video_coords,
audio_coords=audio_coords,
num_frames=ctx.latent_num_frames_for_model,
video_start_frame=video_sp_start or 0,
height=ctx.latent_height,
width=ctx.latent_width,
audio_num_frames=audio_num_frames_latent,
@@ -18,6 +18,7 @@ from sglang.multimodal_gen.runtime.layers.attention import layer
from sglang.multimodal_gen.runtime.layers.attention.backends.sdpa import SDPABackend
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.ltx_2 import (
LTX2AudioVideoRotaryPosEmbed,
_ltx2_build_batched_perturbation_states,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising import (
@@ -52,6 +53,7 @@ class TestLTX2BCGCoords(CustomTestCase):
video_coords=video_coords,
audio_coords=audio_coords,
num_frames=16,
video_start_frame=0,
height=32,
width=48,
audio_num_frames=126,
@@ -70,6 +72,7 @@ class TestLTX2BCGCoords(CustomTestCase):
width=48,
device=self.video_latents.device,
fps=24,
start_frame=0,
)
self.model.audio_rope.prepare_audio_coords.assert_called_once_with(
batch_size=2,
@@ -149,6 +152,57 @@ class TestLTX2BCGCoords(CustomTestCase):
pipeline_config.prepare_video_rope_coords_for_sp.assert_called_once()
pipeline_config.prepare_audio_rope_coords_for_sp.assert_called_once()
def test_legacy_bcg_coords_keep_the_sp_shard_start_frame(self):
rope = LTX2AudioVideoRotaryPosEmbed(dim=64)
model = SimpleNamespace(
rope=rope,
audio_rope=LTX2AudioVideoRotaryPosEmbed(
dim=64, scale_factors=(4,), modality="audio"
),
)
stage = object.__new__(LTX2DenoisingStage)
stage._ltx2_coords_cache = OrderedDict()
stage._ltx2_coords_cache_max_entries = 4
local_frames = 4
ctx = SimpleNamespace(
latent_num_frames_for_model=local_frames,
latent_height=2,
latent_width=3,
use_ltx23_legacy_one_stage=True,
)
server_args = SimpleNamespace(
pipeline_config=SimpleNamespace(), enable_breakable_cuda_graph=True
)
video_latents = torch.zeros(1, local_frames * 2 * 3, 16)
for sp_rank in (0, 1):
with self.subTest(sp_rank=sp_rank):
batch = SimpleNamespace(
fps=24,
did_sp_shard_latents=True,
sp_video_start_frame=sp_rank * local_frames,
did_sp_shard_audio_latents=False,
)
video_coords, _ = stage._get_ltx2_rope_coords(
ctx,
SimpleNamespace(current_model=model),
batch,
server_args,
video_latents,
self.audio_latents,
audio_num_frames_latent=126,
)
# the eager forward shifts each shard by sp_rank * local frames
expected = rope.prepare_video_coords(
batch_size=1,
num_frames=local_frames,
height=2,
width=3,
device=video_latents.device,
fps=24,
start_frame=sp_rank * local_frames,
)
torch.testing.assert_close(video_coords, expected, rtol=0, atol=0)
class TestLTX2GuidanceBCG(CustomTestCase):
@unittest.skipUnless(torch.cuda.is_available(), "CUDA graphs")