From 5b8955e70fc41f457a42f53e9523b1f3d925b867 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 24 Oct 2026 09:17:41 +0800 Subject: [PATCH] [Perf] diffusion: Shard LTX-2 stage 2 over the CFG ranks as Ulysses (#45965) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../runtime/distributed/group_coordinator.py | 19 ++ .../runtime/distributed/parallel_state.py | 31 ++++ .../executors/parallel_executor.py | 14 +- .../runtime/pipelines_core/stages/base.py | 8 + .../ltx_2/denoising_av.py | 24 ++- .../test_sequence_parallel_over_cfg_ranks.py | 171 ++++++++++++++++++ 6 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_sequence_parallel_over_cfg_ranks.py diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py index 471317d49e..8bf51d87bc 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -915,6 +915,25 @@ class SequenceParallelGroupCoordinator(GroupCoordinator): raise RuntimeError( "Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator" ) + self._bind_sequence_groups(ulysses_group, ring_group) + + @classmethod + def over_group( + cls, group: GroupCoordinator, ring_group: ProcessGroup + ) -> "SequenceParallelGroupCoordinator": + """Pure-Ulysses SP over ``group``'s ranks that reuses its communicators. + + ``ring_group`` is this rank's single-rank group. The result owns no + process group: destroying ``group`` releases it, never destroy it itself. + """ + sp_group = cls.__new__(cls) + sp_group.__dict__.update(vars(group)) + sp_group._bind_sequence_groups(group.device_group, ring_group) + return sp_group + + def _bind_sequence_groups( + self, ulysses_group: ProcessGroup, ring_group: ProcessGroup + ) -> None: self.ulysses_group = ulysses_group self.ring_group = ring_group diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py index 4f34019c86..5f3eb75eaf 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py +++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py @@ -68,6 +68,9 @@ _REPLICA: GroupCoordinator | None = None _ENCODER_DP: GroupCoordinator | None = None _SRT_MOE_EP = None _VAE_DECODE: GroupCoordinator | None = None +# the CFG ranks as one Ulysses group for stages that run without CFG; a view +# over _CFG's communicators, built only when cfg > 1 and sp == 1 +_CFG_SP: SequenceParallelGroupCoordinator | None = None _DIT: ProcessGroup | None = None _VAE: ProcessGroup | None = None _VAE_DECODE_PARALLEL_AXES = "tp-sp-pp-cfg" @@ -532,6 +535,11 @@ def initialize_model_parallel( ring_group=PROCESS_GROUP.RING_PG, ) + global _CFG_SP + assert _CFG_SP is None, "CFG sequence parallel group is already initialized" + if classifier_free_guidance_degree > 1 and sequence_parallel_degree == 1: + _CFG_SP = SequenceParallelGroupCoordinator.over_group(_CFG, _SP.ring_group) + global _TP assert _TP is None, "Tensor parallel group is already initialized" _TP = init_parallel_group_coordinator( @@ -614,6 +622,26 @@ def initialize_model_parallel( init_dit_group(dit_parallel_size, backend) +def get_cfg_sequence_parallel_group() -> SequenceParallelGroupCoordinator | None: + return _CFG_SP + + +@contextmanager +def use_sequence_parallel_group(sp_group: SequenceParallelGroupCoordinator): + """Route every SP accessor in the scope to ``sp_group``. + + Latent shards, RoPE offsets and the Ulysses all-to-alls all read the group + through these accessors, so a stage runs entirely on one topology. + """ + global _SP + old_sp_group = _SP + _SP = sp_group + try: + yield + finally: + _SP = old_sp_group + + def get_sp_world_size() -> int: """Return world size for the sequence model parallel group.""" return get_sp_group().world_size @@ -905,6 +933,7 @@ def init_vae_group( def destroy_model_parallel() -> None: """Set the groups to none and destroy them.""" global _TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP + global _CFG_SP _clear_srt_tp_group() # The IPC transport keeps CUDA mappings associated with the current @@ -959,3 +988,5 @@ def destroy_model_parallel() -> None: _TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP = ( None, ) * 10 + # a view over _CFG's communicators, released with it + _CFG_SP = None diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py index 3f80a12807..338bc5456d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/executors/parallel_executor.py @@ -6,9 +6,11 @@ import torch from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_cfg_group, + get_cfg_sequence_parallel_group, get_classifier_free_guidance_rank, get_world_group, get_world_rank, + use_sequence_parallel_group, ) from sglang.multimodal_gen.runtime.distributed.utils import broadcast_pyobj from sglang.multimodal_gen.runtime.pipelines_core import Req @@ -52,7 +54,7 @@ class ParallelExecutor(PipelineExecutor): with self._component_residency_request(stages, batch, server_args): # TODO: decide when to gather on main when CFG_PARALLEL -> MAIN_RANK_ONLY for stage_index, stage in enumerate(stages): - paradigm = stage.parallelism_type + paradigm = stage.parallelism_type_for(batch) if paradigm == StageParallelismType.MAIN_RANK_ONLY: if rank == 0: @@ -120,6 +122,16 @@ class ParallelExecutor(PipelineExecutor): run_stage, use_nvtx, ) + elif paradigm == StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS: + with use_sequence_parallel_group(get_cfg_sequence_parallel_group()): + batch = self._run_stage_with_executor_hooks( + stage, + stage_index, + batch, + server_args, + run_stage, + use_nvtx, + ) elif paradigm == StageParallelismType.MAIN_RANK_ONLY_AND_SEND_TO_OTHERS: obj_list = [] if rank == 0: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py index 2451034162..7a9dcdc98e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -47,6 +47,9 @@ class StageParallelismType(Enum): CFG_PARALLEL = auto() # executed on main rank only and send result to other ranks MAIN_RANK_ONLY_AND_SEND_TO_OTHERS = auto() + # a stage without CFG runs on every CFG rank, sequence-sharded over them + # as one Ulysses group; its inputs must already match across CFG ranks + SEQUENCE_PARALLEL_OVER_CFG_RANKS = auto() class StageVerificationError(Exception): @@ -289,6 +292,11 @@ class PipelineStage(StageDedupMixin, ABC): # return StageParallelismType.MAIN_RANK_ONLY return StageParallelismType.REPLICATED + def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType: + """The parallelism the executor runs ``batch`` with; every rank must + resolve the same type.""" + return self.parallelism_type + def cfg_parallel_local_batch_fields( self, batch: Req, server_args: ServerArgs ) -> tuple[str, ...]: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py index 1cfeaeb58f..c447b5ef6d 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py @@ -5,6 +5,9 @@ from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_world_size, model_parallel_is_initialized, ) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_cfg_sequence_parallel_group, +) from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) @@ -23,6 +26,10 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger logger = init_logger(__name__) +# stage-2 video tokens per rank from which Ulysses over the CFG ranks runs stage 2 +# faster than the main rank alone (LTX-2, 2xH200 eager: +10% at 1920, -7% at 2304) +CFG_SP_MIN_TOKENS_PER_RANK = 2304 + class LTX2AVDenoisingStage(LTX2DenoisingStage): """ @@ -148,8 +155,9 @@ class LTX2RefinementStage(LTX2AVDenoisingStage): @property def parallelism_type(self) -> StageParallelismType: - # Stage 2 is distilled and always runs with CFG disabled, so non-main - # CFG ranks only need the result when the decoder will use all ranks. + # stage 2 is distilled and runs without CFG, so under CFG parallel the + # main rank runs it and sends the result only when the decoder uses + # every rank if self.server_args.enable_cfg_parallel: if ( model_parallel_is_initialized() @@ -161,6 +169,18 @@ class LTX2RefinementStage(LTX2AVDenoisingStage): return StageParallelismType.MAIN_RANK_ONLY return StageParallelismType.REPLICATED + def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType: + # stage 1 leaves the same latents on every CFG rank, so pure-CFG ranks + # can shard stage 2 as one Ulysses group once it is long enough + cfg_sp_group = get_cfg_sequence_parallel_group() + if self.server_args.enable_cfg_parallel and cfg_sp_group is not None: + reqs = batch if isinstance(batch, list) else [batch] + # packed stage-2 video latents [B, S, C], the same shape on every rank + tokens = min(req.latents.shape[1] for req in reqs) + if tokens // cfg_sp_group.world_size >= CFG_SP_MIN_TOKENS_PER_RANK: + return StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS + return self.parallelism_type + @staticmethod def _reset_stage2_generators(batch: Req) -> None: generator = getattr(batch, "generator", None) diff --git a/python/sglang/multimodal_gen/test/unit/test_sequence_parallel_over_cfg_ranks.py b/python/sglang/multimodal_gen/test/unit/test_sequence_parallel_over_cfg_ranks.py new file mode 100644 index 0000000000..ab1fd861e9 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_sequence_parallel_over_cfg_ranks.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A stage without CFG shards a long enough sequence over the CFG ranks as one Ulysses group.""" + +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.multiprocessing as mp +import torch.nn.functional as F + +from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.runtime.distributed import group_coordinator +from sglang.multimodal_gen.runtime.distributed import parallel_state as ps +from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.pipelines_core.executors.parallel_executor import ( + ParallelExecutor, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising_av import ( + CFG_SP_MIN_TOKENS_PER_RANK, + LTX2RefinementStage, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum +from sglang.multimodal_gen.runtime.platforms.cpu import CpuPlatform + + +class _Sdpa: + def forward(self, q, k, v, _ctx): + return F.scaled_dot_product_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) + ).transpose(1, 2) + + +def _ulysses_attention() -> USPAttention: + attn = USPAttention.__new__(USPAttention) + attn.attn_impl = _Sdpa() + attn.backend = AttentionBackendEnum.TORCH_SDPA + attn.skip_sequence_parallel = False + attn.enable_packed_qkv_input_a2a = False + attn.sp_attention_mode = "ulysses" + attn.sp_attention_mode_is_auto = False + return attn + + +def _check_folded_topology(rank, rendezvous): + with patch.object(group_coordinator, "current_platform", CpuPlatform()): + ps.init_distributed_environment( + world_size=2, + rank=rank, + distributed_init_method=rendezvous, + local_rank=rank, + backend="gloo", + timeout=60, + ) + ps.initialize_model_parallel( + classifier_free_guidance_degree=2, + sequence_parallel_degree=1, + backend="gloo", + ) + try: + folded = ps.get_cfg_sequence_parallel_group() + cfg = ps.get_cfg_group() + # a view over the CFG communicators: no new process group per rank + assert folded.device_group is cfg.device_group + assert folded.ulysses_group is cfg.device_group + assert (folded.ranks, folded.rank_in_group) == (cfg.ranks, cfg.rank_in_group) + assert (folded.ulysses_world_size, folded.ulysses_rank) == (2, rank) + assert folded.ring_world_size == 1 + assert ps.get_sp_world_size() == 1 + + stage = object.__new__(LTX2RefinementStage) + stage.server_args = SimpleNamespace(enable_cfg_parallel=True) + stage.vae = None + min_tokens = 2 * CFG_SP_MIN_TOKENS_PER_RANK + short, long = ( + Req(sampling_params=SamplingParams(), latents=torch.empty(1, n, 1)) + for n in (min_tokens - 2, min_tokens) + ) + server_args = SimpleNamespace( + enable_cfg_parallel=True, enable_layerwise_nvtx_marker=False + ) + with ( + patch.object( + ParallelExecutor, + "_component_residency_request", + lambda *args: nullcontext(), + ), + patch.object( + ParallelExecutor, + "_run_stage_with_executor_hooks", + lambda self, stage, index, batch, args, run_stage, nvtx: run_stage( + stage, batch + ), + ), + ): + executor = ParallelExecutor.__new__(ParallelExecutor) + seen_short, seen_long = ( + executor._execute_stages( + [stage], + batch, + server_args, + lambda *_: (ps.get_sp_world_size(), ps.get_sp_parallel_rank()), + ) + for batch in (short, long) + ) + # a short stage 2 runs on the main rank alone; the other keeps its batch + if rank == 0: + assert seen_short == (1, 0) + else: + assert seen_short is short + # a long one runs on every CFG rank inside the folded scope + assert seen_long == (2, rank) + assert ps.get_sp_world_size() == 1 + + torch.manual_seed(0) + config = LTX2PipelineConfig() + tokens_per_frame = 6 + video = torch.randn(1, 16 * tokens_per_frame, 8) + audio = torch.randn(1, 7, 8) + q, k, v = (torch.randn(1, 8, 4, 16) for _ in range(3)) + expected = _Sdpa().forward(q, k, v, None) + local = slice(4 * rank, 4 * rank + 4) + attn = _ulysses_attention() + with ( + ps.use_sequence_parallel_group(folded), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + batch = Req( + height=config.vae_scale_factor * config.patch_size * 2, + width=config.vae_scale_factor * config.patch_size * 3, + ) + video_shard, _ = config.shard_latents_for_sp(batch, video) + audio_shard, _ = config.shard_audio_latents_for_sp(batch, audio) + assert video_shard.shape[1] == 8 * tokens_per_frame + assert batch.sp_video_start_frame == 8 * rank + assert audio_shard.shape[1] == 4 and batch.sp_audio_has_padding + video_gathered = config.gather_latents_for_sp(video_shard, batch) + audio_gathered = config.gather_audio_latents_for_sp(audio_shard, batch) + + out = attn.forward(q[:, local], k[:, local], v[:, local]) + # LTX-2 keeps audio whole under SP: every key appears once per rank, + # so the gathered duplicates leave each softmax unchanged + replicated = attn.forward(q, k, v) + torch.testing.assert_close(video_gathered, video, rtol=0, atol=0) + torch.testing.assert_close(audio_gathered, audio, rtol=0, atol=0) + torch.testing.assert_close(out, expected[:, local]) + torch.testing.assert_close(replicated, expected) + assert ps.get_sp_world_size() == 1 + finally: + ps.destroy_model_parallel() + ps.destroy_distributed_environment() + assert ps.get_cfg_sequence_parallel_group() is None + + +@pytest.mark.skipif( + torch.version.hip is not None, + reason="ROCm builds query GPU 0 while importing sglang, which fails with the " + "GPUs hidden for these CPU workers", +) +def test_stage_without_cfg_shards_over_cfg_ranks(tmp_path, monkeypatch): + # gloo ranks on CPU: hide the GPUs so no communicator binds rank 1 to a + # device a 1-GPU runner lacks, and keep the CUDA-IPC all-to-all off + monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "") + monkeypatch.setenv("SGLANG_DIFFUSION_PLATFORM_OVERRIDE", "cpu") + monkeypatch.setenv("SGLANG_DIFFUSION_IPC_A2A", "0") + rendezvous = (tmp_path / "rendezvous").as_uri() + mp.spawn(_check_folded_topology, args=(rendezvous,), nprocs=2)