[Perf] diffusion: Shard LTX-2 stage 2 over the CFG ranks as Ulysses (#45965)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
738e4535ea
commit
5b8955e70f
@@ -915,6 +915,25 @@ class SequenceParallelGroupCoordinator(GroupCoordinator):
|
||||
raise RuntimeError(
|
||||
"Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator"
|
||||
)
|
||||
self._bind_sequence_groups(ulysses_group, ring_group)
|
||||
|
||||
@classmethod
|
||||
def over_group(
|
||||
cls, group: GroupCoordinator, ring_group: ProcessGroup
|
||||
) -> "SequenceParallelGroupCoordinator":
|
||||
"""Pure-Ulysses SP over ``group``'s ranks that reuses its communicators.
|
||||
|
||||
``ring_group`` is this rank's single-rank group. The result owns no
|
||||
process group: destroying ``group`` releases it, never destroy it itself.
|
||||
"""
|
||||
sp_group = cls.__new__(cls)
|
||||
sp_group.__dict__.update(vars(group))
|
||||
sp_group._bind_sequence_groups(group.device_group, ring_group)
|
||||
return sp_group
|
||||
|
||||
def _bind_sequence_groups(
|
||||
self, ulysses_group: ProcessGroup, ring_group: ProcessGroup
|
||||
) -> None:
|
||||
self.ulysses_group = ulysses_group
|
||||
self.ring_group = ring_group
|
||||
|
||||
|
||||
@@ -68,6 +68,9 @@ _REPLICA: GroupCoordinator | None = None
|
||||
_ENCODER_DP: GroupCoordinator | None = None
|
||||
_SRT_MOE_EP = None
|
||||
_VAE_DECODE: GroupCoordinator | None = None
|
||||
# the CFG ranks as one Ulysses group for stages that run without CFG; a view
|
||||
# over _CFG's communicators, built only when cfg > 1 and sp == 1
|
||||
_CFG_SP: SequenceParallelGroupCoordinator | None = None
|
||||
_DIT: ProcessGroup | None = None
|
||||
_VAE: ProcessGroup | None = None
|
||||
_VAE_DECODE_PARALLEL_AXES = "tp-sp-pp-cfg"
|
||||
@@ -532,6 +535,11 @@ def initialize_model_parallel(
|
||||
ring_group=PROCESS_GROUP.RING_PG,
|
||||
)
|
||||
|
||||
global _CFG_SP
|
||||
assert _CFG_SP is None, "CFG sequence parallel group is already initialized"
|
||||
if classifier_free_guidance_degree > 1 and sequence_parallel_degree == 1:
|
||||
_CFG_SP = SequenceParallelGroupCoordinator.over_group(_CFG, _SP.ring_group)
|
||||
|
||||
global _TP
|
||||
assert _TP is None, "Tensor parallel group is already initialized"
|
||||
_TP = init_parallel_group_coordinator(
|
||||
@@ -614,6 +622,26 @@ def initialize_model_parallel(
|
||||
init_dit_group(dit_parallel_size, backend)
|
||||
|
||||
|
||||
def get_cfg_sequence_parallel_group() -> SequenceParallelGroupCoordinator | None:
|
||||
return _CFG_SP
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_sequence_parallel_group(sp_group: SequenceParallelGroupCoordinator):
|
||||
"""Route every SP accessor in the scope to ``sp_group``.
|
||||
|
||||
Latent shards, RoPE offsets and the Ulysses all-to-alls all read the group
|
||||
through these accessors, so a stage runs entirely on one topology.
|
||||
"""
|
||||
global _SP
|
||||
old_sp_group = _SP
|
||||
_SP = sp_group
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_SP = old_sp_group
|
||||
|
||||
|
||||
def get_sp_world_size() -> int:
|
||||
"""Return world size for the sequence model parallel group."""
|
||||
return get_sp_group().world_size
|
||||
@@ -905,6 +933,7 @@ def init_vae_group(
|
||||
def destroy_model_parallel() -> None:
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP
|
||||
global _CFG_SP
|
||||
|
||||
_clear_srt_tp_group()
|
||||
# The IPC transport keeps CUDA mappings associated with the current
|
||||
@@ -959,3 +988,5 @@ def destroy_model_parallel() -> None:
|
||||
_TP, _SP, _DP, _CFG, _PP, _VAE_DECODE, _DIT, _VAE, _REPLICA, _ENCODER_DP = (
|
||||
None,
|
||||
) * 10
|
||||
# a view over _CFG's communicators, released with it
|
||||
_CFG_SP = None
|
||||
|
||||
@@ -6,9 +6,11 @@ import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_cfg_sequence_parallel_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
get_world_group,
|
||||
get_world_rank,
|
||||
use_sequence_parallel_group,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.utils import broadcast_pyobj
|
||||
from sglang.multimodal_gen.runtime.pipelines_core import Req
|
||||
@@ -52,7 +54,7 @@ class ParallelExecutor(PipelineExecutor):
|
||||
with self._component_residency_request(stages, batch, server_args):
|
||||
# TODO: decide when to gather on main when CFG_PARALLEL -> MAIN_RANK_ONLY
|
||||
for stage_index, stage in enumerate(stages):
|
||||
paradigm = stage.parallelism_type
|
||||
paradigm = stage.parallelism_type_for(batch)
|
||||
|
||||
if paradigm == StageParallelismType.MAIN_RANK_ONLY:
|
||||
if rank == 0:
|
||||
@@ -120,6 +122,16 @@ class ParallelExecutor(PipelineExecutor):
|
||||
run_stage,
|
||||
use_nvtx,
|
||||
)
|
||||
elif paradigm == StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS:
|
||||
with use_sequence_parallel_group(get_cfg_sequence_parallel_group()):
|
||||
batch = self._run_stage_with_executor_hooks(
|
||||
stage,
|
||||
stage_index,
|
||||
batch,
|
||||
server_args,
|
||||
run_stage,
|
||||
use_nvtx,
|
||||
)
|
||||
elif paradigm == StageParallelismType.MAIN_RANK_ONLY_AND_SEND_TO_OTHERS:
|
||||
obj_list = []
|
||||
if rank == 0:
|
||||
|
||||
@@ -47,6 +47,9 @@ class StageParallelismType(Enum):
|
||||
CFG_PARALLEL = auto()
|
||||
# executed on main rank only and send result to other ranks
|
||||
MAIN_RANK_ONLY_AND_SEND_TO_OTHERS = auto()
|
||||
# a stage without CFG runs on every CFG rank, sequence-sharded over them
|
||||
# as one Ulysses group; its inputs must already match across CFG ranks
|
||||
SEQUENCE_PARALLEL_OVER_CFG_RANKS = auto()
|
||||
|
||||
|
||||
class StageVerificationError(Exception):
|
||||
@@ -289,6 +292,11 @@ class PipelineStage(StageDedupMixin, ABC):
|
||||
# return StageParallelismType.MAIN_RANK_ONLY
|
||||
return StageParallelismType.REPLICATED
|
||||
|
||||
def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType:
|
||||
"""The parallelism the executor runs ``batch`` with; every rank must
|
||||
resolve the same type."""
|
||||
return self.parallelism_type
|
||||
|
||||
def cfg_parallel_local_batch_fields(
|
||||
self, batch: Req, server_args: ServerArgs
|
||||
) -> tuple[str, ...]:
|
||||
|
||||
+22
-2
@@ -5,6 +5,9 @@ from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_world_size,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_cfg_sequence_parallel_group,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
@@ -23,6 +26,10 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# stage-2 video tokens per rank from which Ulysses over the CFG ranks runs stage 2
|
||||
# faster than the main rank alone (LTX-2, 2xH200 eager: +10% at 1920, -7% at 2304)
|
||||
CFG_SP_MIN_TOKENS_PER_RANK = 2304
|
||||
|
||||
|
||||
class LTX2AVDenoisingStage(LTX2DenoisingStage):
|
||||
"""
|
||||
@@ -148,8 +155,9 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
|
||||
|
||||
@property
|
||||
def parallelism_type(self) -> StageParallelismType:
|
||||
# Stage 2 is distilled and always runs with CFG disabled, so non-main
|
||||
# CFG ranks only need the result when the decoder will use all ranks.
|
||||
# stage 2 is distilled and runs without CFG, so under CFG parallel the
|
||||
# main rank runs it and sends the result only when the decoder uses
|
||||
# every rank
|
||||
if self.server_args.enable_cfg_parallel:
|
||||
if (
|
||||
model_parallel_is_initialized()
|
||||
@@ -161,6 +169,18 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
|
||||
return StageParallelismType.MAIN_RANK_ONLY
|
||||
return StageParallelismType.REPLICATED
|
||||
|
||||
def parallelism_type_for(self, batch: Req | list[Req]) -> StageParallelismType:
|
||||
# stage 1 leaves the same latents on every CFG rank, so pure-CFG ranks
|
||||
# can shard stage 2 as one Ulysses group once it is long enough
|
||||
cfg_sp_group = get_cfg_sequence_parallel_group()
|
||||
if self.server_args.enable_cfg_parallel and cfg_sp_group is not None:
|
||||
reqs = batch if isinstance(batch, list) else [batch]
|
||||
# packed stage-2 video latents [B, S, C], the same shape on every rank
|
||||
tokens = min(req.latents.shape[1] for req in reqs)
|
||||
if tokens // cfg_sp_group.world_size >= CFG_SP_MIN_TOKENS_PER_RANK:
|
||||
return StageParallelismType.SEQUENCE_PARALLEL_OVER_CFG_RANKS
|
||||
return self.parallelism_type
|
||||
|
||||
@staticmethod
|
||||
def _reset_stage2_generators(batch: Req) -> None:
|
||||
generator = getattr(batch, "generator", None)
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""A stage without CFG shards a long enough sequence over the CFG ranks as one Ulysses group."""
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.distributed import group_coordinator
|
||||
from sglang.multimodal_gen.runtime.distributed import parallel_state as ps
|
||||
from sglang.multimodal_gen.runtime.layers.attention.layer import USPAttention
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.executors.parallel_executor import (
|
||||
ParallelExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.denoising_av import (
|
||||
CFG_SP_MIN_TOKENS_PER_RANK,
|
||||
LTX2RefinementStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.platforms.cpu import CpuPlatform
|
||||
|
||||
|
||||
class _Sdpa:
|
||||
def forward(self, q, k, v, _ctx):
|
||||
return F.scaled_dot_product_attention(
|
||||
q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
|
||||
).transpose(1, 2)
|
||||
|
||||
|
||||
def _ulysses_attention() -> USPAttention:
|
||||
attn = USPAttention.__new__(USPAttention)
|
||||
attn.attn_impl = _Sdpa()
|
||||
attn.backend = AttentionBackendEnum.TORCH_SDPA
|
||||
attn.skip_sequence_parallel = False
|
||||
attn.enable_packed_qkv_input_a2a = False
|
||||
attn.sp_attention_mode = "ulysses"
|
||||
attn.sp_attention_mode_is_auto = False
|
||||
return attn
|
||||
|
||||
|
||||
def _check_folded_topology(rank, rendezvous):
|
||||
with patch.object(group_coordinator, "current_platform", CpuPlatform()):
|
||||
ps.init_distributed_environment(
|
||||
world_size=2,
|
||||
rank=rank,
|
||||
distributed_init_method=rendezvous,
|
||||
local_rank=rank,
|
||||
backend="gloo",
|
||||
timeout=60,
|
||||
)
|
||||
ps.initialize_model_parallel(
|
||||
classifier_free_guidance_degree=2,
|
||||
sequence_parallel_degree=1,
|
||||
backend="gloo",
|
||||
)
|
||||
try:
|
||||
folded = ps.get_cfg_sequence_parallel_group()
|
||||
cfg = ps.get_cfg_group()
|
||||
# a view over the CFG communicators: no new process group per rank
|
||||
assert folded.device_group is cfg.device_group
|
||||
assert folded.ulysses_group is cfg.device_group
|
||||
assert (folded.ranks, folded.rank_in_group) == (cfg.ranks, cfg.rank_in_group)
|
||||
assert (folded.ulysses_world_size, folded.ulysses_rank) == (2, rank)
|
||||
assert folded.ring_world_size == 1
|
||||
assert ps.get_sp_world_size() == 1
|
||||
|
||||
stage = object.__new__(LTX2RefinementStage)
|
||||
stage.server_args = SimpleNamespace(enable_cfg_parallel=True)
|
||||
stage.vae = None
|
||||
min_tokens = 2 * CFG_SP_MIN_TOKENS_PER_RANK
|
||||
short, long = (
|
||||
Req(sampling_params=SamplingParams(), latents=torch.empty(1, n, 1))
|
||||
for n in (min_tokens - 2, min_tokens)
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
enable_cfg_parallel=True, enable_layerwise_nvtx_marker=False
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
ParallelExecutor,
|
||||
"_component_residency_request",
|
||||
lambda *args: nullcontext(),
|
||||
),
|
||||
patch.object(
|
||||
ParallelExecutor,
|
||||
"_run_stage_with_executor_hooks",
|
||||
lambda self, stage, index, batch, args, run_stage, nvtx: run_stage(
|
||||
stage, batch
|
||||
),
|
||||
),
|
||||
):
|
||||
executor = ParallelExecutor.__new__(ParallelExecutor)
|
||||
seen_short, seen_long = (
|
||||
executor._execute_stages(
|
||||
[stage],
|
||||
batch,
|
||||
server_args,
|
||||
lambda *_: (ps.get_sp_world_size(), ps.get_sp_parallel_rank()),
|
||||
)
|
||||
for batch in (short, long)
|
||||
)
|
||||
# a short stage 2 runs on the main rank alone; the other keeps its batch
|
||||
if rank == 0:
|
||||
assert seen_short == (1, 0)
|
||||
else:
|
||||
assert seen_short is short
|
||||
# a long one runs on every CFG rank inside the folded scope
|
||||
assert seen_long == (2, rank)
|
||||
assert ps.get_sp_world_size() == 1
|
||||
|
||||
torch.manual_seed(0)
|
||||
config = LTX2PipelineConfig()
|
||||
tokens_per_frame = 6
|
||||
video = torch.randn(1, 16 * tokens_per_frame, 8)
|
||||
audio = torch.randn(1, 7, 8)
|
||||
q, k, v = (torch.randn(1, 8, 4, 16) for _ in range(3))
|
||||
expected = _Sdpa().forward(q, k, v, None)
|
||||
local = slice(4 * rank, 4 * rank + 4)
|
||||
attn = _ulysses_attention()
|
||||
with (
|
||||
ps.use_sequence_parallel_group(folded),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
batch = Req(
|
||||
height=config.vae_scale_factor * config.patch_size * 2,
|
||||
width=config.vae_scale_factor * config.patch_size * 3,
|
||||
)
|
||||
video_shard, _ = config.shard_latents_for_sp(batch, video)
|
||||
audio_shard, _ = config.shard_audio_latents_for_sp(batch, audio)
|
||||
assert video_shard.shape[1] == 8 * tokens_per_frame
|
||||
assert batch.sp_video_start_frame == 8 * rank
|
||||
assert audio_shard.shape[1] == 4 and batch.sp_audio_has_padding
|
||||
video_gathered = config.gather_latents_for_sp(video_shard, batch)
|
||||
audio_gathered = config.gather_audio_latents_for_sp(audio_shard, batch)
|
||||
|
||||
out = attn.forward(q[:, local], k[:, local], v[:, local])
|
||||
# LTX-2 keeps audio whole under SP: every key appears once per rank,
|
||||
# so the gathered duplicates leave each softmax unchanged
|
||||
replicated = attn.forward(q, k, v)
|
||||
torch.testing.assert_close(video_gathered, video, rtol=0, atol=0)
|
||||
torch.testing.assert_close(audio_gathered, audio, rtol=0, atol=0)
|
||||
torch.testing.assert_close(out, expected[:, local])
|
||||
torch.testing.assert_close(replicated, expected)
|
||||
assert ps.get_sp_world_size() == 1
|
||||
finally:
|
||||
ps.destroy_model_parallel()
|
||||
ps.destroy_distributed_environment()
|
||||
assert ps.get_cfg_sequence_parallel_group() is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.version.hip is not None,
|
||||
reason="ROCm builds query GPU 0 while importing sglang, which fails with the "
|
||||
"GPUs hidden for these CPU workers",
|
||||
)
|
||||
def test_stage_without_cfg_shards_over_cfg_ranks(tmp_path, monkeypatch):
|
||||
# gloo ranks on CPU: hide the GPUs so no communicator binds rank 1 to a
|
||||
# device a 1-GPU runner lacks, and keep the CUDA-IPC all-to-all off
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
|
||||
monkeypatch.setenv("SGLANG_DIFFUSION_PLATFORM_OVERRIDE", "cpu")
|
||||
monkeypatch.setenv("SGLANG_DIFFUSION_IPC_A2A", "0")
|
||||
rendezvous = (tmp_path / "rendezvous").as_uri()
|
||||
mp.spawn(_check_folded_topology, args=(rendezvous,), nprocs=2)
|
||||
Reference in New Issue
Block a user