[diffusion] perf: pipeline MiniMax-H3's Ulysses exchange against attention over the copy engine (#43164)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-09 22:04:29 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent fba3ad4b1f
commit 1203eb9181
9 changed files with 914 additions and 10 deletions
+14
View File
@@ -65,6 +65,14 @@ if TYPE_CHECKING:
# back to NCCL when unavailable. Set 0 to force NCCL. Keep this in step with
# the resolver below -- that is the value the runtime reads.
SGLANG_DIFFUSION_IPC_A2A: bool = True
# copy-engine all-to-all for Ulysses groups of any size on one host; off by
# default while it is validated. Falls back to NCCL when unavailable.
SGLANG_DIFFUSION_IPC_A2A_MULTI: bool = False
# head groups for pipelining MiniMax-H3's Ulysses exchange against dense
# attention over the copy-engine transport: -1 picks a count that divides
# the heads per rank and steps aside when its buffers do not fit, 0 or 1
# keeps the sequential exchange, N >= 2 forces N groups
SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS: int = -1
# a deadlock backstop, not a per-step budget: a rank can legitimately stall
# for seconds (layerwise offload, wan2.2 expert-tower swaps), and expiry now
# retires the transport on every rank and fails the request
@@ -405,6 +413,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Enable cache-dit acceleration for DiT inference
# CUDA-IPC transport for 2-rank Ulysses all-to-all (NVLink same-node)
"SGLANG_DIFFUSION_IPC_A2A": _lazy_bool("SGLANG_DIFFUSION_IPC_A2A", "true"),
"SGLANG_DIFFUSION_IPC_A2A_MULTI": _lazy_bool(
"SGLANG_DIFFUSION_IPC_A2A_MULTI", "false"
),
"SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS": _lazy_int(
"SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS", -1
),
"SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS": _lazy_float(
"SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS", 10000.0
),
@@ -0,0 +1,505 @@
# SPDX-License-Identifier: Apache-2.0
"""Copy-engine transport for N-rank Ulysses all-to-all on one host.
The two-rank ``ipc_a2a`` design widened to every peer: each rank maps the
peers' staging buffers into its own device context and writes its chunk for a
peer straight into that peer's buffer. A contiguous device-to-device copy runs
on the copy engine, so the exchange takes no SMs from the kernels around it --
NCCL's all-to-all is an SM kernel, and next to a full-occupancy attention it
slows that attention by more than the exchange costs.
Per peer, a sequence counter published after the copy orders the data before
the flag, and the receiver spins on that peer's entry. Double-buffered slots
alternate per call; a slot is only rewritten two calls later, after a spin on
every peer, and each peer publishes that call only after it has consumed the
slot in stream order.
"""
import logging
import socket
from collections import OrderedDict
import torch
import torch.distributed as dist
from sglang.kernels.ops.communication.ipc_a2a import load_ipc_a2a_sync
from sglang.kernels.ops.diffusion import pack_qkv_destination_major
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
_Unsupported,
)
logger = logging.getLogger(__name__)
# Group counts tried, in order, when the caller leaves the choice to the
# transport: MiniMax-H3's 56 heads give 28, 14 and 7 per rank at Ulysses 2, 4
# and 8, and 7 groups divide all three.
_AUTO_PIPELINE_GROUPS = (7, 4, 2)
def _member_devices(group, device: int) -> list[int]:
"""Local CUDA ordinal of every member of a same-host group."""
world_size = dist.get_world_size(group=group)
members: list = [None] * world_size
dist.all_gather_object(members, (socket.gethostname(), device), group=group)
if len({host for host, _ in members}) != 1:
raise _Unsupported("requires every Ulysses rank on the same host")
devices = [dev for _, dev in members]
if len(set(devices)) != world_size:
raise _Unsupported(f"Ulysses ranks share CUDA devices {devices}")
return devices
class _StreamMemOps:
"""Front-end signal and wait (cuStreamWriteValue32 / cuStreamWaitValue32).
Neither holds an SM, so a wait queued next to a persistent attention kernel
costs it nothing (a spin kernel would pin an SM and push an attention CTA
into the tail), and a signal queued behind a copy lands when the copy
retires instead of when an SM frees up.
"""
def __init__(self):
import ctypes
cuda = ctypes.CDLL("libcuda.so.1")
args = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint]
self._write = cuda.cuStreamWriteValue32_v2
self._wait = cuda.cuStreamWaitValue32_v2
for fn in (self._write, self._wait):
fn.argtypes = args
fn.restype = ctypes.c_int
# drivers can disable stream memory operations; find out here, inside
# the group's agreed initialization, rather than mid-forward
flag = torch.zeros(1, dtype=torch.int32, device="cuda")
stream = torch.cuda.Stream()
try:
self.write(stream, flag, 1)
self.wait(stream, flag, 1)
except RuntimeError as e:
raise _Unsupported(f"stream memory operations unavailable ({e})") from e
stream.synchronize()
def write(self, stream, flag: torch.Tensor, value: int) -> None:
# default flags fence the stream's earlier writes (the copies) first
rc = self._write(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
if rc:
raise RuntimeError(f"cuStreamWriteValue32 failed with CUresult {rc}")
def wait(self, stream, flag: torch.Tensor, value: int) -> None:
# CU_STREAM_WAIT_VALUE_GEQ compares cyclically, so counters may wrap
rc = self._wait(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
if rc:
raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}")
class _PipelineBuffers:
"""Receive slots and flags for one pipelined-attention shape.
Input slots are group-major ([2, groups, world, chunk]) so a group's rows
form one [S, group_heads, 3 * head_dim] tensor for attention; output slots
are source-major ([2, world, groups, chunk]), each block a peer's rows for
one head group, copied into the merged output as it arrives.
"""
def __init__(self, state, s_local, heads, head_dim, groups, dtype):
world = state.world
self.group_heads = heads // world // groups
rows = s_local * self.group_heads
zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda")
self.inb = state._share(
zeros(2, groups, world, rows * 3 * head_dim), state.group
)
self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group)
# fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g
# peer p has delivered. Groups finish on separate streams, so a single
# output counter could be lowered by a signal that lands late.
self.fin = state._share(zeros(world, dt=torch.int32), state.group)
self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group)
self.send = torch.empty(
world * groups,
s_local,
self.group_heads,
3 * head_dim,
dtype=dtype,
device="cuda",
)
self.calls = 0
class IpcA2AMultiState:
def __init__(self):
self.ops = None
# insertion-ordered so eviction is identical on every rank
self.staging = OrderedDict()
self.flags = None
self.peer_flags = None
self.seq = None
self.timed_out = None
self.peer_timed_out = None
self.budget_ns = 0
self.max_buffers = 0
self.calls = 0
self.rank = None
self.world = 0
self.group = None
self.failed = False
self.inited = False
self.memops = None
self.pipe_in = None
self.pipe_group_streams = []
# pipeline shapes whose buffers did not fit on some rank; agreed by all
self.declined = set()
def reset(self) -> None:
"""Drop mappings that belong to a model-parallel group being replaced."""
self.__init__()
def drop_staging(self) -> None:
if not self.staging:
return
if torch.cuda.is_available():
torch.cuda.synchronize()
self.staging.clear()
def _share(self, t: torch.Tensor, group) -> list[torch.Tensor]:
"""Every member's `t`, peers' re-opened in the LOCAL device context
(a mapping is only dereferenceable from the context that opened it)."""
from torch.multiprocessing.reductions import reduce_tensor
handles: list = [None] * self.world
dist.all_gather_object(handles, reduce_tensor(t), group=group)
dev = torch.cuda.current_device()
out = []
for member, (fn, args) in enumerate(handles):
if member == self.rank:
out.append(t)
continue
args = list(args)
for i, v in enumerate(args):
if isinstance(v, torch.device):
args[i] = torch.device(f"cuda:{dev}")
elif isinstance(v, int) and i == 6:
# rebuild_cuda_tensor positional device index
args[i] = dev
out.append(fn(*args))
return out
def init(self, group) -> None:
import ctypes
self.rank = dist.get_rank(group=group)
self.world = dist.get_world_size(group=group)
self.group = group
dev = torch.cuda.current_device()
devices = _member_devices(group, dev)
peers = [d for member, d in enumerate(devices) if member != self.rank]
missing = [d for d in peers if not torch.cuda.can_device_access_peer(dev, d)]
memops_error = None
try:
memops = _StreamMemOps()
except _Unsupported as e:
memops_error = e
# agree before the handle exchange below: a rank that bailed out here
# alone would leave the others blocked inside that collective
usable = int(not missing and memops_error is None)
agreed = torch.tensor([usable], dtype=torch.int32, device="cuda")
dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group)
if agreed.item() == 0:
if missing:
raise _Unsupported(
f"a Ulysses rank lacks peer-to-peer access (CUDA device {dev} "
f"cannot reach {missing})"
)
raise _Unsupported(
str(memops_error) if memops_error else "a Ulysses peer is unsupported"
)
cudart = ctypes.CDLL("libcudart.so")
for peer_dev in peers:
# kernel-level dereference of peer mappings needs explicit peer access
cudart.cudaDeviceEnablePeerAccess(peer_dev, 0)
self.ops = load_ipc_a2a_sync()
self.memops = memops
# flags[p] is written by peer p; seq[p] counts the calls sent to p
self.flags = torch.zeros(self.world, dtype=torch.int32, device="cuda")
self.seq = torch.zeros(self.world, dtype=torch.int32, device="cuda")
self.timed_out = torch.zeros(1, dtype=torch.int32, device="cuda")
self.budget_ns = int(envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS * 1e6)
self.max_buffers = envs.SGLANG_DIFFUSION_IPC_A2A_MAX_BUFFERS
self.peer_flags = self._share(self.flags, group)
self.peer_timed_out = self._share(self.timed_out, group)
self.inited = True
def get_staging(self, chunk: int, dtype: torch.dtype) -> list[torch.Tensor] | None:
"""Every member's two-slot `[2, world, chunk]` receive buffer (mine at
my rank). Creation is a group collective, so every rank must reach a
new key at the same call site; a miss during capture returns None."""
key = (chunk, dtype)
buffers = self.staging.get(key)
if buffers is None:
if torch.cuda.is_current_stream_capturing():
return None
local = torch.zeros(2, self.world, chunk, dtype=dtype, device="cuda")
buffers = self._share(local, self.group)
self._insert(key, buffers)
return buffers
def _insert(self, key, value) -> None:
if len(self.staging) >= self.max_buffers:
# queued copies may still target the evicted peer mappings
torch.cuda.synchronize()
self.staging.popitem(last=False)
self.staging[key] = value
def pipelined_attention(self, q, k, v, attend, groups: int, fill=None):
"""Ulysses attention with the head exchange pipelined against it.
q/k/v are [s_local, heads, head_dim], sequence-sharded with every head.
``attend(q, k, v)`` runs attention on [S, heads / (world * groups),
head_dim] views of one head group and must treat heads independently,
which keeps the result bit-identical to the sequential exchange.
Group g's copies run on the copy engine while group g-1 attends, and
each group attends on its own stream and sends its output from there,
so no stream waits on a kernel still running on another. Returns
[s_local, heads, head_dim], or None when this call cannot pipeline.
``groups < 0`` picks the count and steps aside, on every rank, when
the buffers would not fit.
``fill(head_start, head_count, dst)``, when given, writes those heads'
q/k/v into a [s_local, head_count, 3 * head_dim] block in place of the
pack (the caller's QK-norm writes its output there); blocks for this
rank go straight into its own receive slot. It is only called once the
call is known to pipeline.
"""
world, r = self.world, self.rank
s_local, heads, head_dim = q.shape
auto = groups < 0
if auto:
groups = next(
(n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0
)
if (
not groups
or heads % (world * groups)
or not (q.shape == k.shape == v.shape)
):
return None
key = ("pipeline", s_local, heads, head_dim, groups, q.dtype)
bufs = self.staging.get(key)
if bufs is None:
if torch.cuda.is_current_stream_capturing() or key in self.declined:
return None
# send + both receive and output slots: 11 * s_local * heads * head_dim
need = 11 * s_local * heads * head_dim * q.element_size()
if auto and not self._fits_on_every_rank(need):
logger.info(
"Ulysses pipeline buffers (%.1f GiB) do not fit; using the "
"sequential exchange for this shape",
need / 2**30,
)
self.declined.add(key)
return None
bufs = _PipelineBuffers(self, s_local, heads, head_dim, groups, q.dtype)
self._insert(key, bufs)
if self.pipe_in is None:
self.pipe_in = torch.cuda.Stream()
while len(self.pipe_group_streams) < groups:
self.pipe_group_streams.append(torch.cuda.Stream())
mem, cin, hg = self.memops, self.pipe_in, bufs.group_heads
main = torch.cuda.current_stream()
slot = bufs.calls % 2
base = bufs.calls * groups
bufs.calls += 1
call = bufs.calls
peers = [(r + step) % world for step in range(1, world)]
own = []
if fill is None:
# block p * groups + g is contiguous: group g of the heads rank p owns
send = pack_qkv_destination_major(q, k, v, world * groups, out=bufs.send)
cin.wait_stream(main)
with torch.cuda.stream(cin):
for g in range(groups):
bufs.inb[r][slot, g, r].copy_(
send[r * groups + g].view(-1), non_blocking=True
)
own.append(cin.record_event())
for p in peers:
bufs.inb[p][slot, g, r].copy_(
send[p * groups + g].view(-1), non_blocking=True
)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
else:
# Group by group: the peers' blocks first, so their copies start
# while this rank fills its own block straight into its slot.
send = bufs.send
block = lambda t: t.view(s_local, hg, 3 * head_dim)
sent = []
for g in range(groups):
for p in peers:
fill((p * groups + g) * hg, hg, block(send[p * groups + g]))
sent.append(main.record_event())
fill((r * groups + g) * hg, hg, block(bufs.inb[r][slot, g, r]))
own.append(main.record_event())
with torch.cuda.stream(cin):
for g in range(groups):
cin.wait_event(sent[g])
for p in peers:
bufs.inb[p][slot, g, r].copy_(
send[p * groups + g].view(-1), non_blocking=True
)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
# Head block b = p * groups + g of the output is group g of rank p's
# heads. Each group stream writes its blocks as soon as they exist, so
# the merge overlaps the groups still attending instead of trailing them.
merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device)
merged_blocks = merged.view(s_local, world * groups, hg, head_dim)
done = []
for g in range(groups):
stream = self.pipe_group_streams[g]
stream.wait_event(own[g])
with torch.cuda.stream(stream):
for p in peers:
mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1)
qkv = bufs.inb[r][slot, g].view(world * s_local, hg, 3 * head_dim)
out = attend(
qkv[..., :head_dim],
qkv[..., head_dim : 2 * head_dim],
qkv[..., 2 * head_dim :],
)
rows = lambda p: out[p * s_local : (p + 1) * s_local]
for p in peers:
bufs.outb[p][slot, r, g].copy_(
rows(p).reshape(-1), non_blocking=True
)
for p in peers:
mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call)
merged_blocks[:, r * groups + g].copy_(rows(r))
for p in peers:
mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call)
merged_blocks[:, p * groups + g].copy_(
bufs.outb[r][slot, p, g].view(s_local, hg, head_dim)
)
done.append(stream.record_event())
for event in done:
main.wait_event(event)
# the next call's pack rewrites `send`, which the copy stream reads
main.wait_stream(cin)
return merged
def _fits_on_every_rank(self, nbytes: int) -> bool:
"""Whether every rank can spare twice `nbytes`, counting the allocator's
cached blocks; a group collective, so all ranks reach the same answer."""
free, _ = torch.cuda.mem_get_info()
cached = torch.cuda.memory_reserved() - torch.cuda.memory_allocated()
fits = torch.tensor(
[int(free + cached >= 2 * nbytes)], dtype=torch.int32, device="cuda"
)
dist.all_reduce(fits, op=dist.ReduceOp.MIN, group=self.group)
return bool(fits.item())
def exchange(self, send: torch.Tensor) -> torch.Tensor | None:
"""``all_to_all_single`` with equal splits: row p of the contiguous
``[world, chunk]`` `send` goes to rank p, and row p of the returned
``[world, chunk]`` view is what rank p sent me."""
if send.shape[0] != self.world or not send.is_contiguous():
return None
buffers = self.get_staging(send[0].numel(), send.dtype)
if buffers is None:
return None
slot = self.calls % 2
self.calls += 1
r = self.rank
buffers[r][slot, r].copy_(send[r].view(-1), non_blocking=True)
# Step k sends to r + k: every step is a permutation, so each rank
# receives from exactly one peer at a time instead of all of them
# converging on the same destination.
peers = [(r + k) % self.world for k in range(1, self.world)]
for p in peers:
buffers[p][slot, r].copy_(send[p].view(-1), non_blocking=True)
self.ops.bump_signal(
self.seq.narrow(0, p, 1), self.peer_flags[p].narrow(0, r, 1)
)
for p in peers:
self.ops.spin_wait(
self.flags.narrow(0, p, 1),
self.seq.narrow(0, p, 1),
self.timed_out,
self.peer_timed_out[p],
self.budget_ns,
)
return buffers[r][slot]
def check_timeout(self) -> None:
"""Retire the transport on every rank if any rank's spin expired.
A spin only flags the peer it waited on, so agree across the whole group
at the request boundary: a rank that retired alone would post an NCCL
all-to-all that its IPC peers never join. Expiry means a peer went
silent for the whole budget and that exchange returned incomplete data,
so this raises instead of serving the result.
"""
if self.failed or not self.inited:
return
expired = self.timed_out.clone()
dist.all_reduce(expired, op=dist.ReduceOp.MAX, group=self.group)
if expired.item() == 0:
return
self.failed = True
raise RuntimeError(
"IPC all-to-all gave up waiting for a peer after "
f"{envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS:g} ms, so that exchange "
"returned incomplete data. The transport is now disabled on every "
"rank; retry the request over NCCL, or set "
"SGLANG_DIFFUSION_IPC_A2A_MULTI=0."
)
IPC_A2A_MULTI = IpcA2AMultiState()
def ipc_a2a_multi_ready(group, enabled: bool | None = None) -> bool:
"""True when the N-rank transport is enabled and initialized on every
member of `group` (initializes lazily on the first eager call)."""
from sglang.multimodal_gen.runtime.platforms import current_platform
if not (envs.SGLANG_DIFFUSION_IPC_A2A_MULTI if enabled is None else enabled):
return False
if IPC_A2A_MULTI.group is not None and IPC_A2A_MULTI.group is not group:
IPC_A2A_MULTI.reset()
if IPC_A2A_MULTI.failed:
return False
if IPC_A2A_MULTI.inited:
return True
if not current_platform.is_cuda() or torch.cuda.is_current_stream_capturing():
return False
ok = True
try:
IPC_A2A_MULTI.init(group)
except _Unsupported as e:
logger.info("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e)
ok = False
except Exception as e:
logger.debug("IPC all-to-all (multi-rank) initialization failed", exc_info=True)
logger.warning("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e)
ok = False
# every member must take the same path, or an IPC rank waits on an NCCL one
agreed = torch.tensor([int(ok)], dtype=torch.int32, device="cuda")
dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group)
if agreed.item() == 0:
IPC_A2A_MULTI.reset()
IPC_A2A_MULTI.failed = True
return False
return True
def ulysses_pipelined_attention(q, k, v, attend, groups: int, fill=None):
"""Pipelined Ulysses attention over the copy-engine transport, or None
when the Ulysses group cannot use it (the caller keeps its NCCL path)."""
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
group = get_sp_group().ulysses_group
if group is None or not ipc_a2a_multi_ready(group, enabled=True):
return None
return IPC_A2A_MULTI.pipelined_attention(q, k, v, attend, groups, fill=fill)
@@ -868,9 +868,11 @@ def destroy_model_parallel() -> None:
# The IPC transport keeps CUDA mappings associated with the current
# Ulysses group. Drop them before tearing down the process groups.
from .device_communicators.ipc_a2a import IPC_A2A
from .device_communicators.ipc_a2a_multi import IPC_A2A_MULTI
from .parallel_groups import PROCESS_GROUP
IPC_A2A.reset()
IPC_A2A_MULTI.reset()
destroyed_groups = []
for group in (
@@ -88,6 +88,17 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
assert ulysses_pg is not None, "Ulysses process group is not initialized."
x_shape = x.shape
x = x.flatten().contiguous()
if x.is_cuda:
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
IPC_A2A_MULTI,
ipc_a2a_multi_ready,
)
if ipc_a2a_multi_ready(ulysses_pg):
world_size = torch.distributed.get_world_size(group=ulysses_pg)
received = IPC_A2A_MULTI.exchange(x.view(world_size, -1))
if received is not None:
return received.view(x_shape)
if role is None:
output = torch.empty_like(x)
else:
@@ -37,6 +37,9 @@ from sglang.multimodal_gen.runtime.distributed import (
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
IPC_A2A,
)
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
IPC_A2A_MULTI,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_cfg_group,
get_classifier_free_guidance_rank,
@@ -559,6 +562,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
# request boundary: the IPC watchdog flag is a device read, illegal
# inside a graph capture and too costly per exchange
IPC_A2A.check_timeout()
IPC_A2A_MULTI.check_timeout()
if len(batch) > 1:
if return_req:
raise ValueError(
@@ -1210,6 +1214,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
from sglang.multimodal_gen.runtime.layers.usp import drop_a2a_staging_buffers
IPC_A2A.drop_staging()
IPC_A2A_MULTI.drop_staging()
drop_a2a_staging_buffers()
torch.get_device_module().empty_cache()
@@ -27,6 +27,7 @@ from sglang.kernels.ops.diffusion import (
can_use_mxfp8_swizzled,
can_use_silu_mul_mxfp8,
fused_inplace_qknorm_rope,
fused_qknorm_rope_out_of_place,
indexed_gate_bf16,
indexed_gate_bf16_,
indexed_scale_shift_bf16_,
@@ -653,6 +654,122 @@ class MiniMaxH3TimeEmbedder(nn.Module):
return out
def _minimax_h3_bind_attention_backend(
attention: MiniMaxH3Attention, dtype: torch.dtype
) -> None:
if attention._attention_impl is None:
attention._set_attention_backend(
get_attn_backend(
attention.head_dim,
dtype,
selected_attention_backend=attention._selected_attention_backend,
attention_requirements=AttentionRequirements(packed_varlen=True),
)
)
def _minimax_h3_pipelined_dense_attention(
attention: MiniMaxH3Attention,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
*,
cu_seqlens: torch.Tensor,
cu_seqlens_host: tuple[int, ...] | None,
max_seqlen: int,
fill=None,
) -> torch.Tensor | None:
"""Dense FA with the Ulysses exchange pipelined over head groups.
Returns None (caller keeps the sequential exchange) unless head-group
pipelining is enabled and the dense FlashAttention path would run.
"""
groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS
if groups in (0, 1) or torch.compiler.is_compiling():
return None
if attention._attention_backend_enum is not AttentionBackendEnum.FA:
return None
impl = attention._attention_impl
if impl._request_skip_softmax_threshold()[0]:
return None
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
ulysses_pipelined_attention,
)
def attend(q_group, k_group, v_group):
return impl.forward_varlen(
q_group,
k_group,
v_group,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
cu_seqlens_host=cu_seqlens_host,
)
return ulysses_pipelined_attention(q, k, v, attend, groups, fill=fill)
def _minimax_h3_qknorm_rope_pipelined_attention(
attention: MiniMaxH3Attention,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
rope_cache: tuple[torch.Tensor, torch.Tensor],
*,
cu_seqlens: torch.Tensor,
cu_seqlens_host: tuple[int, ...] | None,
max_seqlen: int,
) -> torch.Tensor | None:
"""The pipelined attention with QK-norm + RoPE writing into the exchange.
q/k are the raw projections. Each destination block gets its heads'
normalized q/k and v written in place, so the in-place norm and the pack
become one pass. Same kernel arithmetic, so the result is bit-identical.
Returns None, having written nothing, when the call cannot pipeline.
"""
# under graph capture the forward keeps the in-place norm, and the pipeline
# runs from the attention core's eager break point instead
if (
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS in (0, 1)
or torch.cuda.is_current_stream_capturing()
):
return None
_minimax_h3_bind_attention_backend(attention, q.dtype)
cos_sin_cache, positions = rope_cache
head_dim = attention.head_dim
q_weight, k_weight = attention.q_norm.weight, attention.k_norm.weight
def fill(head_start: int, head_count: int, dst: torch.Tensor) -> None:
heads = slice(head_start, head_start + head_count)
fused_qknorm_rope_out_of_place(
q[:, heads],
k[:, heads],
dst[..., :head_dim],
dst[..., head_dim : 2 * head_dim],
q_weight,
k_weight,
cos_sin_cache,
positions,
is_neox=True,
eps=attention.q_norm.eps,
head_dim=head_dim,
rope_dim=cos_sin_cache.shape[-1],
round_norm_before_rope=True,
)
dst[..., 2 * head_dim :].copy_(v[:, heads])
return _minimax_h3_pipelined_dense_attention(
attention,
q,
k,
v,
cu_seqlens=cu_seqlens,
cu_seqlens_host=cu_seqlens_host,
max_seqlen=max_seqlen,
fill=fill,
)
def _minimax_h3_attention_core_impl(
attention: MiniMaxH3Attention,
q: torch.Tensor,
@@ -674,6 +791,21 @@ def _minimax_h3_attention_core_impl(
kernel and sequence-parallel collectives execute eagerly.
"""
_minimax_h3_bind_attention_backend(attention, q.dtype)
if ulysses_active and not ring_active and gate_compress is None:
out = _minimax_h3_pipelined_dense_attention(
attention,
q,
k,
v,
cu_seqlens=cu_seqlens,
cu_seqlens_host=cu_seqlens_host,
max_seqlen=max_seqlen,
)
if out is not None:
return out
if ulysses_active:
from sglang.multimodal_gen.runtime.layers.usp import (
_usp_all_gather,
@@ -683,16 +815,6 @@ def _minimax_h3_attention_core_impl(
q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v)
if attention._attention_impl is None:
attention._set_attention_backend(
get_attn_backend(
attention.head_dim,
q.dtype,
selected_attention_backend=attention._selected_attention_backend,
attention_requirements=AttentionRequirements(packed_varlen=True),
)
)
if attention._attention_backend_enum is AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3:
attn_metadata = (
get_forward_context().attn_metadata
@@ -1159,6 +1281,22 @@ class MiniMaxH3Attention(nn.Module):
else:
cos_sin_cache, positions = rope_cache
if self._use_fused_qknorm_rope and not torch.compiler.is_compiling():
if ulysses_active and not ring_active:
out = _minimax_h3_qknorm_rope_pipelined_attention(
self,
q,
k,
v,
rope_cache,
cu_seqlens=cu_seqlens,
cu_seqlens_host=cu_seqlens_host,
max_seqlen=max_seqlen,
)
if out is not None:
out, _ = self.out_proj(
out.reshape(total, self.num_heads * self.head_dim)
)
return out
fused_inplace_qknorm_rope(
q,
k,
@@ -1561,6 +1561,7 @@ STANDALONE_FILES = {
"../single_test_file/test_disagg_server.py",
"../single_test_file/test_ar_models.py",
"../single_test_file/test_ipc_a2a_2_gpu.py",
"../single_test_file/test_ipc_a2a_multi_gpu.py",
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py",
"../single_test_file/test_encoder_fold_srt_2_gpu.py",
"../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py",
@@ -1602,6 +1603,8 @@ STANDALONE_FILE_EST_TIMES = {
"../single_test_file/test_ar_models.py": 600.0,
# no model load; the cost is the one-time JIT build of the sync kernels
"../single_test_file/test_ipc_a2a_2_gpu.py": 240.0,
# 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT
"../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0,
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0,
"../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0,
# ~60 s locally with a warm HF cache (load + one capture + 4 steps);
@@ -0,0 +1,225 @@
"""The copy-engine all-to-all and the pipelined Ulysses attention must match the
sequential exchange bit for bit.
Both only activate with SGLANG_DIFFUSION_IPC_A2A_MULTI / a pipeline group count
on a same-host group with peer-to-peer access, so nothing in the single-GPU
suite reaches them. Runs on 2 ranks, and on 4 when four GPUs are visible:
pytest -v python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py
"""
from __future__ import annotations
import os
import subprocess
import sys
import unittest
import torch
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.test.test_utils import CustomTestCase
def _attend(q, k, v):
# head-independent and exact under any head grouping; the flips and rolls
# read across the whole gathered sequence, so a misplaced shard shows up
return (q.float() * 2 + k.flip(0).float() - v.roll(1, 0).float()).to(q.dtype)
def _worker() -> int:
import torch.distributed as dist
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
)
rank = int(os.environ["RANK"])
world = int(os.environ["WORLD_SIZE"])
torch.cuda.set_device(rank)
maybe_init_distributed_environment_and_model_parallel(
tp_size=1, sp_size=world, ulysses_degree=world
)
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
IPC_A2A_MULTI,
ulysses_pipelined_attention,
)
from sglang.multimodal_gen.runtime.layers.usp import (
_usp_all_to_all_single,
_usp_input_all_to_all_packed_qkv,
_usp_output_all_to_all,
)
peers = [d for d in range(world) if d != rank]
if not all(torch.cuda.can_device_access_peer(rank, d) for d in peers):
print("SKIP no peer-to-peer access between the devices", flush=True)
return 0
failures = []
def sequential(q, k, v):
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
qs, ks, vs = _usp_input_all_to_all_packed_qkv(q, k, v)
return _usp_output_all_to_all(_attend(qs, ks, vs)[None], head_dim=2)[0]
# (s_local, heads, head_dim, groups); heads % (world * groups) == 0
cases = [(96, 16, 64, 2), (96, 16, 64, 4), (40, 32, 128, 2)]
if world == 2:
cases.append((128, 16, 64, 8))
for s_local, heads, head_dim, groups in cases:
# three calls in a row: the slots alternate and the counters advance
for call in range(3):
torch.manual_seed(
1000 * call + s_local
) # identical full tensors on every rank
full = [
torch.randn(
s_local * world,
heads,
head_dim,
dtype=torch.bfloat16,
device="cuda",
)
for _ in range(3)
]
q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full)
ref = sequential(q, k, v)
got = ulysses_pipelined_attention(q, k, v, _attend, groups)
if got is None:
failures.append(
f"pipelined returned None for {(s_local, heads, head_dim, groups)}"
)
elif not torch.equal(ref, got):
failures.append(
f"pipelined {(s_local, heads, head_dim, groups)} call {call}"
)
# fill: the caller writes each destination block itself (H3's QK-norm does),
# own blocks straight into this rank's receive slot
def transform(t, scale):
return (t.float() * scale + 1).to(t.dtype)
for s_local, heads, head_dim, groups in cases:
for call in range(3):
torch.manual_seed(2000 + 1000 * call + s_local)
full = [
torch.randn(
s_local * world,
heads,
head_dim,
dtype=torch.bfloat16,
device="cuda",
)
for _ in range(3)
]
q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full)
def fill(head_start, head_count, dst):
h = slice(head_start, head_start + head_count)
dst[..., :head_dim].copy_(transform(q[:, h], 2))
dst[..., head_dim : 2 * head_dim].copy_(transform(k[:, h], 3))
dst[..., 2 * head_dim :].copy_(v[:, h])
ref = sequential(transform(q, 2), transform(k, 3), v)
got = ulysses_pipelined_attention(q, k, v, _attend, groups, fill=fill)
if got is None:
failures.append(
f"filled pipeline returned None for {(s_local, heads, head_dim, groups)}"
)
elif not torch.equal(ref, got):
failures.append(
f"filled pipeline {(s_local, heads, head_dim, groups)} call {call}"
)
# groups=-1 picks a count that divides the heads per rank (4 for 16 heads at
# 2 ranks, 2 at 4) and still matches
torch.manual_seed(77)
full = [
torch.randn(64 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
for _ in range(3)
]
q, k, v = (t.narrow(0, rank * 64, 64).contiguous() for t in full)
got = ulysses_pipelined_attention(q, k, v, _attend, -1)
if got is None or not torch.equal(sequential(q, k, v), got):
failures.append("auto group count")
# a head count the groups cannot split must decline, not mis-shard
q = torch.randn(32, 12, 64, dtype=torch.bfloat16, device="cuda")
if ulysses_pipelined_attention(q, q, q, _attend, 4) is not None:
failures.append("pipelined accepted 12 heads in 4 groups")
# the plain N-rank exchange behind _usp_all_to_all_single
for numel in (world * 1024, world * 4096 * 33):
x = torch.randn(numel, dtype=torch.bfloat16, device="cuda")
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
ref = _usp_all_to_all_single(x.clone())
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = True
calls = IPC_A2A_MULTI.calls
got = _usp_all_to_all_single(x.clone())
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
if IPC_A2A_MULTI.calls == calls:
failures.append(f"exchange of {numel} elements never took the copy engine")
elif not torch.equal(ref, got):
failures.append(f"exchange of {numel} elements")
# a comparison where every arm quietly fell back would pass while testing nothing
if not IPC_A2A_MULTI.inited or IPC_A2A_MULTI.failed or not IPC_A2A_MULTI.staging:
failures.append(
f"transport never engaged (inited={IPC_A2A_MULTI.inited} "
f"failed={IPC_A2A_MULTI.failed} staged={len(IPC_A2A_MULTI.staging)})"
)
verdict = torch.tensor([len(failures)], device="cuda")
dist.all_reduce(verdict)
if failures:
print(f"rank{rank} MISMATCH: {failures}", flush=True)
if rank == 0:
print(
f"IPC_A2A_MULTI_PARITY {'FAIL' if verdict.item() else 'PASS'} world={world}",
flush=True,
)
dist.barrier()
dist.destroy_process_group()
return 1 if verdict.item() else 0
class TestIpcA2AMultiGpu(CustomTestCase):
def _run(self, world: int, port: int):
if not current_platform.is_cuda():
self.skipTest("CUDA-IPC transport is unavailable on this platform")
if torch.cuda.device_count() < world:
self.skipTest(f"needs {world} GPUs")
proc = subprocess.run(
[
sys.executable,
"-m",
"torch.distributed.run",
f"--nproc-per-node={world}",
f"--master-port={port}",
__file__,
"--worker",
],
capture_output=True,
text=True,
timeout=1200,
)
print(proc.stdout[-4000:])
if proc.returncode != 0:
print(proc.stderr[-4000:], file=sys.stderr)
self.assertEqual(proc.returncode, 0, "copy-engine all-to-all diverged")
if "SKIP" not in proc.stdout:
self.assertIn(f"IPC_A2A_MULTI_PARITY PASS world={world}", proc.stdout)
def test_two_ranks_match_sequential_bitwise(self):
self._run(2, 29527)
def test_four_ranks_match_sequential_bitwise(self):
self._run(4, 29537)
if __name__ == "__main__":
if "--worker" in sys.argv:
raise SystemExit(_worker())
unittest.main()