[Perf] diffusion: Shard LTX-2 stage 2 over the CFG ranks as Ulysses (#45965)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-24 09:17:41 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 738e4535ea
commit 5b8955e70f
6 changed files with 264 additions and 3 deletions
@@ -915,6 +915,25 @@ class SequenceParallelGroupCoordinator(GroupCoordinator):
raise RuntimeError(
"Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator"
)
self._bind_sequence_groups(ulysses_group, ring_group)
@classmethod
def over_group(
cls, group: GroupCoordinator, ring_group: ProcessGroup
) -> "SequenceParallelGroupCoordinator":
"""Pure-Ulysses SP over ``group``'s ranks that reuses its communicators.
``ring_group`` is this rank's single-rank group. The result owns no
process group: destroying ``group`` releases it, never destroy it itself.
"""
sp_group = cls.__new__(cls)
sp_group.__dict__.update(vars(group))
sp_group._bind_sequence_groups(group.device_group, ring_group)
return sp_group
def _bind_sequence_groups(
self, ulysses_group: ProcessGroup, ring_group: ProcessGroup
) -> None:
self.ulysses_group = ulysses_group
self.ring_group = ring_group
@@ -68,6 +68,9 @@ _REPLICA: GroupCoordinator | None = None
_ENCODER_DP: GroupCoordinator | None = None
_SRT_MOE_EP = None
_VAE_DECODE: GroupCoordinator | None = None
# the CFG ranks as one Ulysses group for stages that run without CFG; a view
# over _CFG's communicators, built only when cfg > 1 and sp == 1
_CFG_SP: SequenceParallelGroupCoordinator | None = None
_DIT: ProcessGroup | None = None
_VAE: ProcessGroup | None = None
_VAE_DECODE_PARALLEL_AXES = "tp-sp-pp-cfg"
@@ -532,6 +535,11 @@ def initialize_model_parallel(
ring_group=PROCESS_GROUP.RING_PG,
)
global _CFG_SP
assert _CFG_SP is None, "CFG sequence parallel group is already initialized"
if classifier_free_guidance_degree > 1 and sequence_parallel_degree == 1:
_CFG_SP = SequenceParallelGroupCoordinator.over_group(_CFG, _SP.ring_group)
global _TP
assert _TP is None, "Tensor parallel group is already initialized"
_TP = init_parallel_group_coordinator(
@@ -614,6 +622,26 @@ def initialize_model_parallel(
init_dit_group(dit_parallel_size, backend)
def get_cfg_sequence_parallel_group() -> SequenceParallelGroupCoordinator | None:
return _CFG_SP
@contextmanager
def use_sequence_parallel_group(sp_group: SequenceParallelGroupCoordinator):
"""Route every SP accessor in the scope to ``sp_group``.
Latent shards, RoPE offsets and the Ulysses all-to-alls all read the group
through these accessors, so a stage runs entirely on one topology.
"""
global _SP
old_sp_group = _SP
_SP = sp_group
try:
yield
finally:
_SP = old_sp_group
def get_sp_world_size() -> int:
"""Return world size for the sequence model parallel group."""
return get_sp_group().world_size
@@ -905,6 +933,7 @@ def init_vae_group(
def destroy_model_parallel() -> None:
"""Set the groups to none and destroy them."""
global _TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP
global _CFG_SP
_clear_srt_tp_group()
# The IPC transport keeps CUDA mappings associated with the current
@@ -959,3 +988,5 @@ def destroy_model_parallel() -> None:
_TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP = (
None,
) * 10
# a view over _CFG's communicators, released with it
_CFG_SP = None
@@ -6,9 +6,11 @@ import torch
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_cfg_group,
get_cfg_sequence_parallel_group,
get_classifier_free_guidance_rank,
get_world_group,
get_world_rank,
use_sequence_parallel_group,
)
from sglang.multimodal_gen.runtime.distributed.utils import broadcast_pyobj
from sglang.multimodal_gen.runtime.pipelines_core import Req
@@ -52,7 +54,7 @@ class ParallelExecutor(PipelineExecutor):
with self._component_residency_request(stages, batch, server_args):
# TODO: decide when to gather on main when CFG_PARALLEL -> MAIN_RANK_ONLY
for stage_index, stage in enumerate(stages):
paradigm = stage.parallelism_type
paradigm = stage.parallelism_type_for(batch)
if paradigm == StageParallelismType.MAIN_RANK_ONLY:
if rank == 0:
@@ -120,6 +122,16 @@ class ParallelExecutor(PipelineExecutor):
run_stage,
use_nvtx,
)
elif paradigm == StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS:
with use_sequence_parallel_group(get_cfg_sequence_parallel_group()):
batch = self._run_stage_with_executor_hooks(
stage,
stage_index,
batch,
server_args,
run_stage,
use_nvtx,
)
elif paradigm == StageParallelismType.MAIN_RANK_ONLY_AND_SEND_TO_OTHERS:
obj_list = []
if rank == 0:
@@ -47,6 +47,9 @@ class StageParallelismType(Enum):
CFG_PARALLEL = auto()
# executed on main rank only and send result to other ranks
MAIN_RANK_ONLY_AND_SEND_TO_OTHERS = auto()
# a stage without CFG runs on every CFG rank, sequence-sharded over them
# as one Ulysses group; its inputs must already match across CFG ranks
SEQUENCE_PARALLEL_OVER_CFG_RANKS = auto()
class StageVerificationError(Exception):
@@ -289,6 +292,11 @@ class PipelineStage(StageDedupMixin, ABC):
# return StageParallelismType.MAIN_RANK_ONLY
return StageParallelismType.REPLICATED
def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType:
"""The parallelism the executor runs ``batch`` with; every rank must
resolve the same type."""
return self.parallelism_type
def cfg_parallel_local_batch_fields(
self, batch: Req, server_args: ServerArgs
) -> tuple[str, ...]:
@@ -5,6 +5,9 @@ from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_world_size,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_cfg_sequence_parallel_group,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
@@ -23,6 +26,10 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
# stage-2 video tokens per rank from which Ulysses over the CFG ranks runs stage 2
# faster than the main rank alone (LTX-2, 2xH200 eager: +10% at 1920, -7% at 2304)
CFG_SP_MIN_TOKENS_PER_RANK = 2304
class LTX2AVDenoisingStage(LTX2DenoisingStage):
"""
@@ -148,8 +155,9 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
@property
def parallelism_type(self) -> StageParallelismType:
# Stage 2 is distilled and always runs with CFG disabled, so non-main
# CFG ranks only need the result when the decoder will use all ranks.
# stage 2 is distilled and runs without CFG, so under CFG parallel the
# main rank runs it and sends the result only when the decoder uses
# every rank
if self.server_args.enable_cfg_parallel:
if (
model_parallel_is_initialized()
@@ -161,6 +169,18 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
return StageParallelismType.MAIN_RANK_ONLY
return StageParallelismType.REPLICATED
def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType:
# stage 1 leaves the same latents on every CFG rank, so pure-CFG ranks
# can shard stage 2 as one Ulysses group once it is long enough
cfg_sp_group = get_cfg_sequence_parallel_group()
if self.server_args.enable_cfg_parallel and cfg_sp_group is not None:
reqs = batch if isinstance(batch, list) else [batch]
# packed stage-2 video latents [B, S, C], the same shape on every rank
tokens = min(req.latents.shape[1] for req in reqs)
if tokens // cfg_sp_group.world_size >= CFG_SP_MIN_TOKENS_PER_RANK:
return StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS
return self.parallelism_type
@staticmethod
def _reset_stage2_generators(batch: Req) -> None:
generator = getattr(batch, "generator", None)
@@ -0,0 +1,171 @@
# SPDX-License-Identifier: Apache-2.0
"""A stage without CFG shards a long enough sequence over the CFG ranks as one Ulysses group."""
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.multiprocessing as mp
import torch.nn.functional as F
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.distributed import group_coordinator
from sglang.multimodal_gen.runtime.distributed import parallel_state as ps
from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.pipelines_core.executors.parallel_executor import (
ParallelExecutor,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising_av import (
CFG_SP_MIN_TOKENS_PER_RANK,
LTX2RefinementStage,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.platforms.cpu import CpuPlatform
class _Sdpa:
def forward(self, q, k, v, _ctx):
return F.scaled_dot_product_attention(
q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
).transpose(1, 2)
def _ulysses_attention() -> USPAttention:
attn = USPAttention.__new__(USPAttention)
attn.attn_impl = _Sdpa()
attn.backend = AttentionBackendEnum.TORCH_SDPA
attn.skip_sequence_parallel = False
attn.enable_packed_qkv_input_a2a = False
attn.sp_attention_mode = "ulysses"
attn.sp_attention_mode_is_auto = False
return attn
def _check_folded_topology(rank, rendezvous):
with patch.object(group_coordinator, "current_platform", CpuPlatform()):
ps.init_distributed_environment(
world_size=2,
rank=rank,
distributed_init_method=rendezvous,
local_rank=rank,
backend="gloo",
timeout=60,
)
ps.initialize_model_parallel(
classifier_free_guidance_degree=2,
sequence_parallel_degree=1,
backend="gloo",
)
try:
folded = ps.get_cfg_sequence_parallel_group()
cfg = ps.get_cfg_group()
# a view over the CFG communicators: no new process group per rank
assert folded.device_group is cfg.device_group
assert folded.ulysses_group is cfg.device_group
assert (folded.ranks, folded.rank_in_group) == (cfg.ranks, cfg.rank_in_group)
assert (folded.ulysses_world_size, folded.ulysses_rank) == (2, rank)
assert folded.ring_world_size == 1
assert ps.get_sp_world_size() == 1
stage = object.__new__(LTX2RefinementStage)
stage.server_args = SimpleNamespace(enable_cfg_parallel=True)
stage.vae = None
min_tokens = 2 * CFG_SP_MIN_TOKENS_PER_RANK
short, long = (
Req(sampling_params=SamplingParams(), latents=torch.empty(1, n, 1))
for n in (min_tokens - 2, min_tokens)
)
server_args = SimpleNamespace(
enable_cfg_parallel=True, enable_layerwise_nvtx_marker=False
)
with (
patch.object(
ParallelExecutor,
"_component_residency_request",
lambda *args: nullcontext(),
),
patch.object(
ParallelExecutor,
"_run_stage_with_executor_hooks",
lambda self, stage, index, batch, args, run_stage, nvtx: run_stage(
stage, batch
),
),
):
executor = ParallelExecutor.__new__(ParallelExecutor)
seen_short, seen_long = (
executor._execute_stages(
[stage],
batch,
server_args,
lambda *_: (ps.get_sp_world_size(), ps.get_sp_parallel_rank()),
)
for batch in (short, long)
)
# a short stage 2 runs on the main rank alone; the other keeps its batch
if rank == 0:
assert seen_short == (1, 0)
else:
assert seen_short is short
# a long one runs on every CFG rank inside the folded scope
assert seen_long == (2, rank)
assert ps.get_sp_world_size() == 1
torch.manual_seed(0)
config = LTX2PipelineConfig()
tokens_per_frame = 6
video = torch.randn(1, 16 * tokens_per_frame, 8)
audio = torch.randn(1, 7, 8)
q, k, v = (torch.randn(1, 8, 4, 16) for _ in range(3))
expected = _Sdpa().forward(q, k, v, None)
local = slice(4 * rank, 4 * rank + 4)
attn = _ulysses_attention()
with (
ps.use_sequence_parallel_group(folded),
set_forward_context(current_timestep=0, attn_metadata=None),
):
batch = Req(
height=config.vae_scale_factor * config.patch_size * 2,
width=config.vae_scale_factor * config.patch_size * 3,
)
video_shard, _ = config.shard_latents_for_sp(batch, video)
audio_shard, _ = config.shard_audio_latents_for_sp(batch, audio)
assert video_shard.shape[1] == 8 * tokens_per_frame
assert batch.sp_video_start_frame == 8 * rank
assert audio_shard.shape[1] == 4 and batch.sp_audio_has_padding
video_gathered = config.gather_latents_for_sp(video_shard, batch)
audio_gathered = config.gather_audio_latents_for_sp(audio_shard, batch)
out = attn.forward(q[:, local], k[:, local], v[:, local])
# LTX-2 keeps audio whole under SP: every key appears once per rank,
# so the gathered duplicates leave each softmax unchanged
replicated = attn.forward(q, k, v)
torch.testing.assert_close(video_gathered, video, rtol=0, atol=0)
torch.testing.assert_close(audio_gathered, audio, rtol=0, atol=0)
torch.testing.assert_close(out, expected[:, local])
torch.testing.assert_close(replicated, expected)
assert ps.get_sp_world_size() == 1
finally:
ps.destroy_model_parallel()
ps.destroy_distributed_environment()
assert ps.get_cfg_sequence_parallel_group() is None
@pytest.mark.skipif(
torch.version.hip is not None,
reason="ROCm builds query GPU 0 while importing sglang, which fails with the "
"GPUs hidden for these CPU workers",
)
def test_stage_without_cfg_shards_over_cfg_ranks(tmp_path, monkeypatch):
# gloo ranks on CPU: hide the GPUs so no communicator binds rank 1 to a
# device a 1-GPU runner lacks, and keep the CUDA-IPC all-to-all off
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
monkeypatch.setenv("SGLANG_DIFFUSION_PLATFORM_OVERRIDE", "cpu")
monkeypatch.setenv("SGLANG_DIFFUSION_IPC_A2A", "0")
rendezvous = (tmp_path / "rendezvous").as_uri()
mp.spawn(_check_folded_topology, args=(rendezvous,), nprocs=2)