From 1203eb918127a370a9ece4c33593df62467a15b0 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 9 Oct 2026 22:04:29 +0800 Subject: [PATCH] [diffusion] perf: pipeline MiniMax-H3's Ulysses exchange against attention over the copy engine (#43164) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 1 + python/sglang/multimodal_gen/envs.py | 14 + .../device_communicators/ipc_a2a_multi.py | 505 ++++++++++++++++++ .../runtime/distributed/parallel_state.py | 2 + .../multimodal_gen/runtime/layers/usp.py | 11 + .../runtime/managers/gpu_worker.py | 5 + .../runtime/models/dits/minimax_h3.py | 158 +++++- .../multimodal_gen/test/server/gpu_cases.py | 3 + .../test_ipc_a2a_multi_gpu.py | 225 ++++++++ 9 files changed, 914 insertions(+), 10 deletions(-) create mode 100644 python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a_multi.py create mode 100644 python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 67032d58db..015ea4301c 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -1468,6 +1468,7 @@ fold decision is not node-boundary aware: - The released visual VAE quality recipe uses overlapping tiled decode. SGLang keeps that recipe by default and distributes complete tiles across the decode group; this changes scheduling, not the computation inside each tile. - H3 rejects `--vae-config.parallel-decode-mode spatial` and `spatial_shard`: validation found output mismatches. Use the default released tiled recipe. - Keep the default `--encoder-parallel auto`. With the server’s default `batching_max_size` of 1, single-node H100/H200/B200/B300 recipes with peer-to-peer access fold the Qwen text encoder over otherwise idle Ulysses ranks. This is separate from DiT tensor parallelism. A pure-TP recipe already shards the encoder over its TP group and does not add a world fold. +- Single-node Ulysses recipes pipeline each block's all-to-all against dense FlashAttention over the copy engine: one head group's q/k/v moves while the previous group attends, and the output is bit-identical to the sequential exchange. It needs peer-to-peer access and stream memory operations on every rank and about 11 × (rows per rank) × heads × head_dim extra activations per rank (3.9 GB for Ref2VA at Ulysses 2); when that does not fit on some rank, every rank keeps the sequential exchange. `SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS=0` restores the sequential exchange. - For throughput-oriented serving, select **DP (batched throughput)**. The picker pairs `--encoder-parallel dp` with an editable `--batching-max-size` greater than 1. Encoder DP stays inside each DiT replica and composes with encoder TP: the H100 TP2 + Ulysses2 recipe has two TP-sharded encoder copies that can split a batch, while the RTX 5090 pure-TP2 recipe has one encoder copy and therefore no additional batch-DP degree. It provides no benefit for a batch of one and is not bitwise-identical to the unsplit deployment. - Use explicit **Fold** to prioritize single-request latency and encoder memory on a measured high-bandwidth single-node topology. Use **Replicate** as the compatibility path when folding or encoder DP is unsuitable. - `--use-fsdp-inference true` shards only the DiT. MiniMax-H3 preserves the original FP32 dtype of its patch, time, and output projections during FSDP all-gather, so this path does not trade numerical correctness for memory. On 4×H100, prefer TP2 + Ulysses2 for speed; use FSDP as an explicit capacity policy rather than assuming it is faster. diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 4d20d27861..d78d980512 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -65,6 +65,14 @@ if TYPE_CHECKING: # back to NCCL when unavailable. Set 0 to force NCCL. Keep this in step with # the resolver below -- that is the value the runtime reads. SGLANG_DIFFUSION_IPC_A2A: bool = True + # copy-engine all-to-all for Ulysses groups of any size on one host; off by + # default while it is validated. Falls back to NCCL when unavailable. + SGLANG_DIFFUSION_IPC_A2A_MULTI: bool = False + # head groups for pipelining MiniMax-H3's Ulysses exchange against dense + # attention over the copy-engine transport: -1 picks a count that divides + # the heads per rank and steps aside when its buffers do not fit, 0 or 1 + # keeps the sequential exchange, N >= 2 forces N groups + SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS: int = -1 # a deadlock backstop, not a per-step budget: a rank can legitimately stall # for seconds (layerwise offload, wan2.2 expert-tower swaps), and expiry now # retires the transport on every rank and fails the request @@ -405,6 +413,12 @@ environment_variables: dict[str, Callable[[], Any]] = { # Enable cache-dit acceleration for DiT inference # CUDA-IPC transport for 2-rank Ulysses all-to-all (NVLink same-node) "SGLANG_DIFFUSION_IPC_A2A": _lazy_bool("SGLANG_DIFFUSION_IPC_A2A", "true"), + "SGLANG_DIFFUSION_IPC_A2A_MULTI": _lazy_bool( + "SGLANG_DIFFUSION_IPC_A2A_MULTI", "false" + ), + "SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS": _lazy_int( + "SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS", -1 + ), "SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS": _lazy_float( "SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS", 10000.0 ), diff --git a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a_multi.py b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a_multi.py new file mode 100644 index 0000000000..1cc0650ca0 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/ipc_a2a_multi.py @@ -0,0 +1,505 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Copy-engine transport for N-rank Ulysses all-to-all on one host. + +The two-rank ``ipc_a2a`` design widened to every peer: each rank maps the +peers' staging buffers into its own device context and writes its chunk for a +peer straight into that peer's buffer. A contiguous device-to-device copy runs +on the copy engine, so the exchange takes no SMs from the kernels around it -- +NCCL's all-to-all is an SM kernel, and next to a full-occupancy attention it +slows that attention by more than the exchange costs. + +Per peer, a sequence counter published after the copy orders the data before +the flag, and the receiver spins on that peer's entry. Double-buffered slots +alternate per call; a slot is only rewritten two calls later, after a spin on +every peer, and each peer publishes that call only after it has consumed the +slot in stream order. +""" + +import logging +import socket +from collections import OrderedDict + +import torch +import torch.distributed as dist + +from sglang.kernels.ops.communication.ipc_a2a import load_ipc_a2a_sync +from sglang.kernels.ops.diffusion import pack_qkv_destination_major +from sglang.multimodal_gen import envs +from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import ( + _Unsupported, +) + +logger = logging.getLogger(__name__) + +# Group counts tried, in order, when the caller leaves the choice to the +# transport: MiniMax-H3's 56 heads give 28, 14 and 7 per rank at Ulysses 2, 4 +# and 8, and 7 groups divide all three. +_AUTO_PIPELINE_GROUPS = (7, 4, 2) + + +def _member_devices(group, device: int) -> list[int]: + """Local CUDA ordinal of every member of a same-host group.""" + world_size = dist.get_world_size(group=group) + members: list = [None] * world_size + dist.all_gather_object(members, (socket.gethostname(), device), group=group) + if len({host for host, _ in members}) != 1: + raise _Unsupported("requires every Ulysses rank on the same host") + devices = [dev for _, dev in members] + if len(set(devices)) != world_size: + raise _Unsupported(f"Ulysses ranks share CUDA devices {devices}") + return devices + + +class _StreamMemOps: + """Front-end signal and wait (cuStreamWriteValue32 / cuStreamWaitValue32). + + Neither holds an SM, so a wait queued next to a persistent attention kernel + costs it nothing (a spin kernel would pin an SM and push an attention CTA + into the tail), and a signal queued behind a copy lands when the copy + retires instead of when an SM frees up. + """ + + def __init__(self): + import ctypes + + cuda = ctypes.CDLL("libcuda.so.1") + args = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint] + self._write = cuda.cuStreamWriteValue32_v2 + self._wait = cuda.cuStreamWaitValue32_v2 + for fn in (self._write, self._wait): + fn.argtypes = args + fn.restype = ctypes.c_int + # drivers can disable stream memory operations; find out here, inside + # the group's agreed initialization, rather than mid-forward + flag = torch.zeros(1, dtype=torch.int32, device="cuda") + stream = torch.cuda.Stream() + try: + self.write(stream, flag, 1) + self.wait(stream, flag, 1) + except RuntimeError as e: + raise _Unsupported(f"stream memory operations unavailable ({e})") from e + stream.synchronize() + + def write(self, stream, flag: torch.Tensor, value: int) -> None: + # default flags fence the stream's earlier writes (the copies) first + rc = self._write(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0) + if rc: + raise RuntimeError(f"cuStreamWriteValue32 failed with CUresult {rc}") + + def wait(self, stream, flag: torch.Tensor, value: int) -> None: + # CU_STREAM_WAIT_VALUE_GEQ compares cyclically, so counters may wrap + rc = self._wait(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0) + if rc: + raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}") + + +class _PipelineBuffers: + """Receive slots and flags for one pipelined-attention shape. + + Input slots are group-major ([2, groups, world, chunk]) so a group's rows + form one [S, group_heads, 3 * head_dim] tensor for attention; output slots + are source-major ([2, world, groups, chunk]), each block a peer's rows for + one head group, copied into the merged output as it arrives. + """ + + def __init__(self, state, s_local, heads, head_dim, groups, dtype): + world = state.world + self.group_heads = heads // world // groups + rows = s_local * self.group_heads + zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda") + self.inb = state._share( + zeros(2, groups, world, rows * 3 * head_dim), state.group + ) + self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group) + # fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g + # peer p has delivered. Groups finish on separate streams, so a single + # output counter could be lowered by a signal that lands late. + self.fin = state._share(zeros(world, dt=torch.int32), state.group) + self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group) + self.send = torch.empty( + world * groups, + s_local, + self.group_heads, + 3 * head_dim, + dtype=dtype, + device="cuda", + ) + self.calls = 0 + + +class IpcA2AMultiState: + def __init__(self): + self.ops = None + # insertion-ordered so eviction is identical on every rank + self.staging = OrderedDict() + self.flags = None + self.peer_flags = None + self.seq = None + self.timed_out = None + self.peer_timed_out = None + self.budget_ns = 0 + self.max_buffers = 0 + self.calls = 0 + self.rank = None + self.world = 0 + self.group = None + self.failed = False + self.inited = False + self.memops = None + self.pipe_in = None + self.pipe_group_streams = [] + # pipeline shapes whose buffers did not fit on some rank; agreed by all + self.declined = set() + + def reset(self) -> None: + """Drop mappings that belong to a model-parallel group being replaced.""" + self.__init__() + + def drop_staging(self) -> None: + if not self.staging: + return + if torch.cuda.is_available(): + torch.cuda.synchronize() + self.staging.clear() + + def _share(self, t: torch.Tensor, group) -> list[torch.Tensor]: + """Every member's `t`, peers' re-opened in the LOCAL device context + (a mapping is only dereferenceable from the context that opened it).""" + from torch.multiprocessing.reductions import reduce_tensor + + handles: list = [None] * self.world + dist.all_gather_object(handles, reduce_tensor(t), group=group) + dev = torch.cuda.current_device() + out = [] + for member, (fn, args) in enumerate(handles): + if member == self.rank: + out.append(t) + continue + args = list(args) + for i, v in enumerate(args): + if isinstance(v, torch.device): + args[i] = torch.device(f"cuda:{dev}") + elif isinstance(v, int) and i == 6: + # rebuild_cuda_tensor positional device index + args[i] = dev + out.append(fn(*args)) + return out + + def init(self, group) -> None: + import ctypes + + self.rank = dist.get_rank(group=group) + self.world = dist.get_world_size(group=group) + self.group = group + dev = torch.cuda.current_device() + devices = _member_devices(group, dev) + peers = [d for member, d in enumerate(devices) if member != self.rank] + missing = [d for d in peers if not torch.cuda.can_device_access_peer(dev, d)] + memops_error = None + try: + memops = _StreamMemOps() + except _Unsupported as e: + memops_error = e + # agree before the handle exchange below: a rank that bailed out here + # alone would leave the others blocked inside that collective + usable = int(not missing and memops_error is None) + agreed = torch.tensor([usable], dtype=torch.int32, device="cuda") + dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group) + if agreed.item() == 0: + if missing: + raise _Unsupported( + f"a Ulysses rank lacks peer-to-peer access (CUDA device {dev} " + f"cannot reach {missing})" + ) + raise _Unsupported( + str(memops_error) if memops_error else "a Ulysses peer is unsupported" + ) + cudart = ctypes.CDLL("libcudart.so") + for peer_dev in peers: + # kernel-level dereference of peer mappings needs explicit peer access + cudart.cudaDeviceEnablePeerAccess(peer_dev, 0) + self.ops = load_ipc_a2a_sync() + self.memops = memops + # flags[p] is written by peer p; seq[p] counts the calls sent to p + self.flags = torch.zeros(self.world, dtype=torch.int32, device="cuda") + self.seq = torch.zeros(self.world, dtype=torch.int32, device="cuda") + self.timed_out = torch.zeros(1, dtype=torch.int32, device="cuda") + self.budget_ns = int(envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS * 1e6) + self.max_buffers = envs.SGLANG_DIFFUSION_IPC_A2A_MAX_BUFFERS + self.peer_flags = self._share(self.flags, group) + self.peer_timed_out = self._share(self.timed_out, group) + self.inited = True + + def get_staging(self, chunk: int, dtype: torch.dtype) -> list[torch.Tensor] | None: + """Every member's two-slot `[2, world, chunk]` receive buffer (mine at + my rank). Creation is a group collective, so every rank must reach a + new key at the same call site; a miss during capture returns None.""" + key = (chunk, dtype) + buffers = self.staging.get(key) + if buffers is None: + if torch.cuda.is_current_stream_capturing(): + return None + local = torch.zeros(2, self.world, chunk, dtype=dtype, device="cuda") + buffers = self._share(local, self.group) + self._insert(key, buffers) + return buffers + + def _insert(self, key, value) -> None: + if len(self.staging) >= self.max_buffers: + # queued copies may still target the evicted peer mappings + torch.cuda.synchronize() + self.staging.popitem(last=False) + self.staging[key] = value + + def pipelined_attention(self, q, k, v, attend, groups: int, fill=None): + """Ulysses attention with the head exchange pipelined against it. + + q/k/v are [s_local, heads, head_dim], sequence-sharded with every head. + ``attend(q, k, v)`` runs attention on [S, heads / (world * groups), + head_dim] views of one head group and must treat heads independently, + which keeps the result bit-identical to the sequential exchange. + Group g's copies run on the copy engine while group g-1 attends, and + each group attends on its own stream and sends its output from there, + so no stream waits on a kernel still running on another. Returns + [s_local, heads, head_dim], or None when this call cannot pipeline. + ``groups < 0`` picks the count and steps aside, on every rank, when + the buffers would not fit. + + ``fill(head_start, head_count, dst)``, when given, writes those heads' + q/k/v into a [s_local, head_count, 3 * head_dim] block in place of the + pack (the caller's QK-norm writes its output there); blocks for this + rank go straight into its own receive slot. It is only called once the + call is known to pipeline. + """ + world, r = self.world, self.rank + s_local, heads, head_dim = q.shape + auto = groups < 0 + if auto: + groups = next( + (n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0 + ) + if ( + not groups + or heads % (world * groups) + or not (q.shape == k.shape == v.shape) + ): + return None + key = ("pipeline", s_local, heads, head_dim, groups, q.dtype) + bufs = self.staging.get(key) + if bufs is None: + if torch.cuda.is_current_stream_capturing() or key in self.declined: + return None + # send + both receive and output slots: 11 * s_local * heads * head_dim + need = 11 * s_local * heads * head_dim * q.element_size() + if auto and not self._fits_on_every_rank(need): + logger.info( + "Ulysses pipeline buffers (%.1f GiB) do not fit; using the " + "sequential exchange for this shape", + need / 2**30, + ) + self.declined.add(key) + return None + bufs = _PipelineBuffers(self, s_local, heads, head_dim, groups, q.dtype) + self._insert(key, bufs) + if self.pipe_in is None: + self.pipe_in = torch.cuda.Stream() + while len(self.pipe_group_streams) < groups: + self.pipe_group_streams.append(torch.cuda.Stream()) + mem, cin, hg = self.memops, self.pipe_in, bufs.group_heads + main = torch.cuda.current_stream() + slot = bufs.calls % 2 + base = bufs.calls * groups + bufs.calls += 1 + call = bufs.calls + peers = [(r + step) % world for step in range(1, world)] + own = [] + if fill is None: + # block p * groups + g is contiguous: group g of the heads rank p owns + send = pack_qkv_destination_major(q, k, v, world * groups, out=bufs.send) + cin.wait_stream(main) + with torch.cuda.stream(cin): + for g in range(groups): + bufs.inb[r][slot, g, r].copy_( + send[r * groups + g].view(-1), non_blocking=True + ) + own.append(cin.record_event()) + for p in peers: + bufs.inb[p][slot, g, r].copy_( + send[p * groups + g].view(-1), non_blocking=True + ) + for p in peers: + mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1) + else: + # Group by group: the peers' blocks first, so their copies start + # while this rank fills its own block straight into its slot. + send = bufs.send + block = lambda t: t.view(s_local, hg, 3 * head_dim) + sent = [] + for g in range(groups): + for p in peers: + fill((p * groups + g) * hg, hg, block(send[p * groups + g])) + sent.append(main.record_event()) + fill((r * groups + g) * hg, hg, block(bufs.inb[r][slot, g, r])) + own.append(main.record_event()) + with torch.cuda.stream(cin): + for g in range(groups): + cin.wait_event(sent[g]) + for p in peers: + bufs.inb[p][slot, g, r].copy_( + send[p * groups + g].view(-1), non_blocking=True + ) + for p in peers: + mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1) + # Head block b = p * groups + g of the output is group g of rank p's + # heads. Each group stream writes its blocks as soon as they exist, so + # the merge overlaps the groups still attending instead of trailing them. + merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device) + merged_blocks = merged.view(s_local, world * groups, hg, head_dim) + done = [] + for g in range(groups): + stream = self.pipe_group_streams[g] + stream.wait_event(own[g]) + with torch.cuda.stream(stream): + for p in peers: + mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1) + qkv = bufs.inb[r][slot, g].view(world * s_local, hg, 3 * head_dim) + out = attend( + qkv[..., :head_dim], + qkv[..., head_dim : 2 * head_dim], + qkv[..., 2 * head_dim :], + ) + rows = lambda p: out[p * s_local : (p + 1) * s_local] + for p in peers: + bufs.outb[p][slot, r, g].copy_( + rows(p).reshape(-1), non_blocking=True + ) + for p in peers: + mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call) + merged_blocks[:, r * groups + g].copy_(rows(r)) + for p in peers: + mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call) + merged_blocks[:, p * groups + g].copy_( + bufs.outb[r][slot, p, g].view(s_local, hg, head_dim) + ) + done.append(stream.record_event()) + for event in done: + main.wait_event(event) + # the next call's pack rewrites `send`, which the copy stream reads + main.wait_stream(cin) + return merged + + def _fits_on_every_rank(self, nbytes: int) -> bool: + """Whether every rank can spare twice `nbytes`, counting the allocator's + cached blocks; a group collective, so all ranks reach the same answer.""" + free, _ = torch.cuda.mem_get_info() + cached = torch.cuda.memory_reserved() - torch.cuda.memory_allocated() + fits = torch.tensor( + [int(free + cached >= 2 * nbytes)], dtype=torch.int32, device="cuda" + ) + dist.all_reduce(fits, op=dist.ReduceOp.MIN, group=self.group) + return bool(fits.item()) + + def exchange(self, send: torch.Tensor) -> torch.Tensor | None: + """``all_to_all_single`` with equal splits: row p of the contiguous + ``[world, chunk]`` `send` goes to rank p, and row p of the returned + ``[world, chunk]`` view is what rank p sent me.""" + if send.shape[0] != self.world or not send.is_contiguous(): + return None + buffers = self.get_staging(send[0].numel(), send.dtype) + if buffers is None: + return None + slot = self.calls % 2 + self.calls += 1 + r = self.rank + buffers[r][slot, r].copy_(send[r].view(-1), non_blocking=True) + # Step k sends to r + k: every step is a permutation, so each rank + # receives from exactly one peer at a time instead of all of them + # converging on the same destination. + peers = [(r + k) % self.world for k in range(1, self.world)] + for p in peers: + buffers[p][slot, r].copy_(send[p].view(-1), non_blocking=True) + self.ops.bump_signal( + self.seq.narrow(0, p, 1), self.peer_flags[p].narrow(0, r, 1) + ) + for p in peers: + self.ops.spin_wait( + self.flags.narrow(0, p, 1), + self.seq.narrow(0, p, 1), + self.timed_out, + self.peer_timed_out[p], + self.budget_ns, + ) + return buffers[r][slot] + + def check_timeout(self) -> None: + """Retire the transport on every rank if any rank's spin expired. + + A spin only flags the peer it waited on, so agree across the whole group + at the request boundary: a rank that retired alone would post an NCCL + all-to-all that its IPC peers never join. Expiry means a peer went + silent for the whole budget and that exchange returned incomplete data, + so this raises instead of serving the result. + """ + if self.failed or not self.inited: + return + expired = self.timed_out.clone() + dist.all_reduce(expired, op=dist.ReduceOp.MAX, group=self.group) + if expired.item() == 0: + return + self.failed = True + raise RuntimeError( + "IPC all-to-all gave up waiting for a peer after " + f"{envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS:g} ms, so that exchange " + "returned incomplete data. The transport is now disabled on every " + "rank; retry the request over NCCL, or set " + "SGLANG_DIFFUSION_IPC_A2A_MULTI=0." + ) + + +IPC_A2A_MULTI = IpcA2AMultiState() + + +def ipc_a2a_multi_ready(group, enabled: bool | None = None) -> bool: + """True when the N-rank transport is enabled and initialized on every + member of `group` (initializes lazily on the first eager call).""" + from sglang.multimodal_gen.runtime.platforms import current_platform + + if not (envs.SGLANG_DIFFUSION_IPC_A2A_MULTI if enabled is None else enabled): + return False + if IPC_A2A_MULTI.group is not None and IPC_A2A_MULTI.group is not group: + IPC_A2A_MULTI.reset() + if IPC_A2A_MULTI.failed: + return False + if IPC_A2A_MULTI.inited: + return True + if not current_platform.is_cuda() or torch.cuda.is_current_stream_capturing(): + return False + ok = True + try: + IPC_A2A_MULTI.init(group) + except _Unsupported as e: + logger.info("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e) + ok = False + except Exception as e: + logger.debug("IPC all-to-all (multi-rank) initialization failed", exc_info=True) + logger.warning("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e) + ok = False + # every member must take the same path, or an IPC rank waits on an NCCL one + agreed = torch.tensor([int(ok)], dtype=torch.int32, device="cuda") + dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group) + if agreed.item() == 0: + IPC_A2A_MULTI.reset() + IPC_A2A_MULTI.failed = True + return False + return True + + +def ulysses_pipelined_attention(q, k, v, attend, groups: int, fill=None): + """Pipelined Ulysses attention over the copy-engine transport, or None + when the Ulysses group cannot use it (the caller keeps its NCCL path).""" + from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group + + group = get_sp_group().ulysses_group + if group is None or not ipc_a2a_multi_ready(group, enabled=True): + return None + return IPC_A2A_MULTI.pipelined_attention(q, k, v, attend, groups, fill=fill) diff --git a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py index 826ae957fb..99fcb918f4 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py +++ b/python/sglang/multimodal_gen/runtime/distributed/parallel_state.py @@ -868,9 +868,11 @@ def destroy_model_parallel() -> None: # The IPC transport keeps CUDA mappings associated with the current # Ulysses group. Drop them before tearing down the process groups. from .device_communicators.ipc_a2a import IPC_A2A + from .device_communicators.ipc_a2a_multi import IPC_A2A_MULTI from .parallel_groups import PROCESS_GROUP IPC_A2A.reset() + IPC_A2A_MULTI.reset() destroyed_groups = [] for group in ( diff --git a/python/sglang/multimodal_gen/runtime/layers/usp.py b/python/sglang/multimodal_gen/runtime/layers/usp.py index 6be91d9404..8103c4ce16 100644 --- a/python/sglang/multimodal_gen/runtime/layers/usp.py +++ b/python/sglang/multimodal_gen/runtime/layers/usp.py @@ -88,6 +88,17 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te assert ulysses_pg is not None, "Ulysses process group is not initialized." x_shape = x.shape x = x.flatten().contiguous() + if x.is_cuda: + from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( + IPC_A2A_MULTI, + ipc_a2a_multi_ready, + ) + + if ipc_a2a_multi_ready(ulysses_pg): + world_size = torch.distributed.get_world_size(group=ulysses_pg) + received = IPC_A2A_MULTI.exchange(x.view(world_size, -1)) + if received is not None: + return received.view(x_shape) if role is None: output = torch.empty_like(x) else: diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index 11daed7066..6113fd1c04 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -37,6 +37,9 @@ from sglang.multimodal_gen.runtime.distributed import ( from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import ( IPC_A2A, ) +from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( + IPC_A2A_MULTI, +) from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_cfg_group, get_classifier_free_guidance_rank, @@ -559,6 +562,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): # request boundary: the IPC watchdog flag is a device read, illegal # inside a graph capture and too costly per exchange IPC_A2A.check_timeout() + IPC_A2A_MULTI.check_timeout() if len(batch) > 1: if return_req: raise ValueError( @@ -1210,6 +1214,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): from sglang.multimodal_gen.runtime.layers.usp import drop_a2a_staging_buffers IPC_A2A.drop_staging() + IPC_A2A_MULTI.drop_staging() drop_a2a_staging_buffers() torch.get_device_module().empty_cache() diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index 782b79a223..2a19b8931e 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -27,6 +27,7 @@ from sglang.kernels.ops.diffusion import ( can_use_mxfp8_swizzled, can_use_silu_mul_mxfp8, fused_inplace_qknorm_rope, + fused_qknorm_rope_out_of_place, indexed_gate_bf16, indexed_gate_bf16_, indexed_scale_shift_bf16_, @@ -653,6 +654,122 @@ class MiniMaxH3TimeEmbedder(nn.Module): return out +def _minimax_h3_bind_attention_backend( + attention: MiniMaxH3Attention, dtype: torch.dtype +) -> None: + if attention._attention_impl is None: + attention._set_attention_backend( + get_attn_backend( + attention.head_dim, + dtype, + selected_attention_backend=attention._selected_attention_backend, + attention_requirements=AttentionRequirements(packed_varlen=True), + ) + ) + + +def _minimax_h3_pipelined_dense_attention( + attention: MiniMaxH3Attention, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + cu_seqlens: torch.Tensor, + cu_seqlens_host: tuple[int, ...] | None, + max_seqlen: int, + fill=None, +) -> torch.Tensor | None: + """Dense FA with the Ulysses exchange pipelined over head groups. + + Returns None (caller keeps the sequential exchange) unless head-group + pipelining is enabled and the dense FlashAttention path would run. + """ + groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS + if groups in (0, 1) or torch.compiler.is_compiling(): + return None + if attention._attention_backend_enum is not AttentionBackendEnum.FA: + return None + impl = attention._attention_impl + if impl._request_skip_softmax_threshold()[0]: + return None + from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( + ulysses_pipelined_attention, + ) + + def attend(q_group, k_group, v_group): + return impl.forward_varlen( + q_group, + k_group, + v_group, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + cu_seqlens_host=cu_seqlens_host, + ) + + return ulysses_pipelined_attention(q, k, v, attend, groups, fill=fill) + + +def _minimax_h3_qknorm_rope_pipelined_attention( + attention: MiniMaxH3Attention, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + rope_cache: tuple[torch.Tensor, torch.Tensor], + *, + cu_seqlens: torch.Tensor, + cu_seqlens_host: tuple[int, ...] | None, + max_seqlen: int, +) -> torch.Tensor | None: + """The pipelined attention with QK-norm + RoPE writing into the exchange. + + q/k are the raw projections. Each destination block gets its heads' + normalized q/k and v written in place, so the in-place norm and the pack + become one pass. Same kernel arithmetic, so the result is bit-identical. + Returns None, having written nothing, when the call cannot pipeline. + """ + # under graph capture the forward keeps the in-place norm, and the pipeline + # runs from the attention core's eager break point instead + if ( + envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS in (0, 1) + or torch.cuda.is_current_stream_capturing() + ): + return None + _minimax_h3_bind_attention_backend(attention, q.dtype) + cos_sin_cache, positions = rope_cache + head_dim = attention.head_dim + q_weight, k_weight = attention.q_norm.weight, attention.k_norm.weight + + def fill(head_start: int, head_count: int, dst: torch.Tensor) -> None: + heads = slice(head_start, head_start + head_count) + fused_qknorm_rope_out_of_place( + q[:, heads], + k[:, heads], + dst[..., :head_dim], + dst[..., head_dim : 2 * head_dim], + q_weight, + k_weight, + cos_sin_cache, + positions, + is_neox=True, + eps=attention.q_norm.eps, + head_dim=head_dim, + rope_dim=cos_sin_cache.shape[-1], + round_norm_before_rope=True, + ) + dst[..., 2 * head_dim :].copy_(v[:, heads]) + + return _minimax_h3_pipelined_dense_attention( + attention, + q, + k, + v, + cu_seqlens=cu_seqlens, + cu_seqlens_host=cu_seqlens_host, + max_seqlen=max_seqlen, + fill=fill, + ) + + def _minimax_h3_attention_core_impl( attention: MiniMaxH3Attention, q: torch.Tensor, @@ -674,6 +791,21 @@ def _minimax_h3_attention_core_impl( kernel and sequence-parallel collectives execute eagerly. """ + _minimax_h3_bind_attention_backend(attention, q.dtype) + + if ulysses_active and not ring_active and gate_compress is None: + out = _minimax_h3_pipelined_dense_attention( + attention, + q, + k, + v, + cu_seqlens=cu_seqlens, + cu_seqlens_host=cu_seqlens_host, + max_seqlen=max_seqlen, + ) + if out is not None: + return out + if ulysses_active: from sglang.multimodal_gen.runtime.layers.usp import ( _usp_all_gather, @@ -683,16 +815,6 @@ def _minimax_h3_attention_core_impl( q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v) - if attention._attention_impl is None: - attention._set_attention_backend( - get_attn_backend( - attention.head_dim, - q.dtype, - selected_attention_backend=attention._selected_attention_backend, - attention_requirements=AttentionRequirements(packed_varlen=True), - ) - ) - if attention._attention_backend_enum is AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3: attn_metadata = ( get_forward_context().attn_metadata @@ -1159,6 +1281,22 @@ class MiniMaxH3Attention(nn.Module): else: cos_sin_cache, positions = rope_cache if self._use_fused_qknorm_rope and not torch.compiler.is_compiling(): + if ulysses_active and not ring_active: + out = _minimax_h3_qknorm_rope_pipelined_attention( + self, + q, + k, + v, + rope_cache, + cu_seqlens=cu_seqlens, + cu_seqlens_host=cu_seqlens_host, + max_seqlen=max_seqlen, + ) + if out is not None: + out, _ = self.out_proj( + out.reshape(total, self.num_heads * self.head_dim) + ) + return out fused_inplace_qknorm_rope( q, k, diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index d4385f8861..4ea2a0e2f8 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -1561,6 +1561,7 @@ STANDALONE_FILES = { "../single_test_file/test_disagg_server.py", "../single_test_file/test_ar_models.py", "../single_test_file/test_ipc_a2a_2_gpu.py", + "../single_test_file/test_ipc_a2a_multi_gpu.py", "../single_test_file/test_encoder_fold_srt_linear_2_gpu.py", "../single_test_file/test_encoder_fold_srt_2_gpu.py", "../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py", @@ -1602,6 +1603,8 @@ STANDALONE_FILE_EST_TIMES = { "../single_test_file/test_ar_models.py": 600.0, # no model load; the cost is the one-time JIT build of the sync kernels "../single_test_file/test_ipc_a2a_2_gpu.py": 240.0, + # 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT + "../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0, "../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0, "../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0, # ~60 s locally with a warm HF cache (load + one capture + 4 steps); diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py b/python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py new file mode 100644 index 0000000000..abe2a132df --- /dev/null +++ b/python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py @@ -0,0 +1,225 @@ +"""The copy-engine all-to-all and the pipelined Ulysses attention must match the +sequential exchange bit for bit. + +Both only activate with SGLANG_DIFFUSION_IPC_A2A_MULTI / a pipeline group count +on a same-host group with peer-to-peer access, so nothing in the single-GPU +suite reaches them. Runs on 2 ranks, and on 4 when four GPUs are visible: + + pytest -v python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import unittest + +import torch + +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.test.test_utils import CustomTestCase + + +def _attend(q, k, v): + # head-independent and exact under any head grouping; the flips and rolls + # read across the whole gathered sequence, so a misplaced shard shows up + return (q.float() * 2 + k.flip(0).float() - v.roll(1, 0).float()).to(q.dtype) + + +def _worker() -> int: + import torch.distributed as dist + + from sglang.multimodal_gen import envs + from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + ) + + rank = int(os.environ["RANK"]) + world = int(os.environ["WORLD_SIZE"]) + torch.cuda.set_device(rank) + maybe_init_distributed_environment_and_model_parallel( + tp_size=1, sp_size=world, ulysses_degree=world + ) + + from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import ( + IPC_A2A_MULTI, + ulysses_pipelined_attention, + ) + from sglang.multimodal_gen.runtime.layers.usp import ( + _usp_all_to_all_single, + _usp_input_all_to_all_packed_qkv, + _usp_output_all_to_all, + ) + + peers = [d for d in range(world) if d != rank] + if not all(torch.cuda.can_device_access_peer(rank, d) for d in peers): + print("SKIP no peer-to-peer access between the devices", flush=True) + return 0 + + failures = [] + + def sequential(q, k, v): + envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False + qs, ks, vs = _usp_input_all_to_all_packed_qkv(q, k, v) + return _usp_output_all_to_all(_attend(qs, ks, vs)[None], head_dim=2)[0] + + # (s_local, heads, head_dim, groups); heads % (world * groups) == 0 + cases = [(96, 16, 64, 2), (96, 16, 64, 4), (40, 32, 128, 2)] + if world == 2: + cases.append((128, 16, 64, 8)) + for s_local, heads, head_dim, groups in cases: + # three calls in a row: the slots alternate and the counters advance + for call in range(3): + torch.manual_seed( + 1000 * call + s_local + ) # identical full tensors on every rank + full = [ + torch.randn( + s_local * world, + heads, + head_dim, + dtype=torch.bfloat16, + device="cuda", + ) + for _ in range(3) + ] + q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full) + ref = sequential(q, k, v) + got = ulysses_pipelined_attention(q, k, v, _attend, groups) + if got is None: + failures.append( + f"pipelined returned None for {(s_local, heads, head_dim, groups)}" + ) + elif not torch.equal(ref, got): + failures.append( + f"pipelined {(s_local, heads, head_dim, groups)} call {call}" + ) + + # fill: the caller writes each destination block itself (H3's QK-norm does), + # own blocks straight into this rank's receive slot + def transform(t, scale): + return (t.float() * scale + 1).to(t.dtype) + + for s_local, heads, head_dim, groups in cases: + for call in range(3): + torch.manual_seed(2000 + 1000 * call + s_local) + full = [ + torch.randn( + s_local * world, + heads, + head_dim, + dtype=torch.bfloat16, + device="cuda", + ) + for _ in range(3) + ] + q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full) + + def fill(head_start, head_count, dst): + h = slice(head_start, head_start + head_count) + dst[..., :head_dim].copy_(transform(q[:, h], 2)) + dst[..., head_dim : 2 * head_dim].copy_(transform(k[:, h], 3)) + dst[..., 2 * head_dim :].copy_(v[:, h]) + + ref = sequential(transform(q, 2), transform(k, 3), v) + got = ulysses_pipelined_attention(q, k, v, _attend, groups, fill=fill) + if got is None: + failures.append( + f"filled pipeline returned None for {(s_local, heads, head_dim, groups)}" + ) + elif not torch.equal(ref, got): + failures.append( + f"filled pipeline {(s_local, heads, head_dim, groups)} call {call}" + ) + + # groups=-1 picks a count that divides the heads per rank (4 for 16 heads at + # 2 ranks, 2 at 4) and still matches + torch.manual_seed(77) + full = [ + torch.randn(64 * world, 16, 64, dtype=torch.bfloat16, device="cuda") + for _ in range(3) + ] + q, k, v = (t.narrow(0, rank * 64, 64).contiguous() for t in full) + got = ulysses_pipelined_attention(q, k, v, _attend, -1) + if got is None or not torch.equal(sequential(q, k, v), got): + failures.append("auto group count") + + # a head count the groups cannot split must decline, not mis-shard + q = torch.randn(32, 12, 64, dtype=torch.bfloat16, device="cuda") + if ulysses_pipelined_attention(q, q, q, _attend, 4) is not None: + failures.append("pipelined accepted 12 heads in 4 groups") + + # the plain N-rank exchange behind _usp_all_to_all_single + for numel in (world * 1024, world * 4096 * 33): + x = torch.randn(numel, dtype=torch.bfloat16, device="cuda") + envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False + ref = _usp_all_to_all_single(x.clone()) + envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = True + calls = IPC_A2A_MULTI.calls + got = _usp_all_to_all_single(x.clone()) + envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False + if IPC_A2A_MULTI.calls == calls: + failures.append(f"exchange of {numel} elements never took the copy engine") + elif not torch.equal(ref, got): + failures.append(f"exchange of {numel} elements") + + # a comparison where every arm quietly fell back would pass while testing nothing + if not IPC_A2A_MULTI.inited or IPC_A2A_MULTI.failed or not IPC_A2A_MULTI.staging: + failures.append( + f"transport never engaged (inited={IPC_A2A_MULTI.inited} " + f"failed={IPC_A2A_MULTI.failed} staged={len(IPC_A2A_MULTI.staging)})" + ) + + verdict = torch.tensor([len(failures)], device="cuda") + dist.all_reduce(verdict) + if failures: + print(f"rank{rank} MISMATCH: {failures}", flush=True) + if rank == 0: + print( + f"IPC_A2A_MULTI_PARITY {'FAIL' if verdict.item() else 'PASS'} world={world}", + flush=True, + ) + dist.barrier() + dist.destroy_process_group() + return 1 if verdict.item() else 0 + + +class TestIpcA2AMultiGpu(CustomTestCase): + def _run(self, world: int, port: int): + if not current_platform.is_cuda(): + self.skipTest("CUDA-IPC transport is unavailable on this platform") + if torch.cuda.device_count() < world: + self.skipTest(f"needs {world} GPUs") + proc = subprocess.run( + [ + sys.executable, + "-m", + "torch.distributed.run", + f"--nproc-per-node={world}", + f"--master-port={port}", + __file__, + "--worker", + ], + capture_output=True, + text=True, + timeout=1200, + ) + print(proc.stdout[-4000:]) + if proc.returncode != 0: + print(proc.stderr[-4000:], file=sys.stderr) + self.assertEqual(proc.returncode, 0, "copy-engine all-to-all diverged") + if "SKIP" not in proc.stdout: + self.assertIn(f"IPC_A2A_MULTI_PARITY PASS world={world}", proc.stdout) + + def test_two_ranks_match_sequential_bitwise(self): + self._run(2, 29527) + + def test_four_ranks_match_sequential_bitwise(self): + self._run(4, 29537) + + +if __name__ == "__main__": + if "--worker" in sys.argv: + raise SystemExit(_worker()) + unittest.main()