[diffusion] perf: pipeline MiniMax-H3's Ulysses exchange against attention over the copy engine (#43164)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
fba3ad4b1f
commit
1203eb9181
@@ -65,6 +65,14 @@ if TYPE_CHECKING:
|
||||
# back to NCCL when unavailable. Set 0 to force NCCL. Keep this in step with
|
||||
# the resolver below -- that is the value the runtime reads.
|
||||
SGLANG_DIFFUSION_IPC_A2A: bool = True
|
||||
# copy-engine all-to-all for Ulysses groups of any size on one host; off by
|
||||
# default while it is validated. Falls back to NCCL when unavailable.
|
||||
SGLANG_DIFFUSION_IPC_A2A_MULTI: bool = False
|
||||
# head groups for pipelining MiniMax-H3's Ulysses exchange against dense
|
||||
# attention over the copy-engine transport: -1 picks a count that divides
|
||||
# the heads per rank and steps aside when its buffers do not fit, 0 or 1
|
||||
# keeps the sequential exchange, N >= 2 forces N groups
|
||||
SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS: int = -1
|
||||
# a deadlock backstop, not a per-step budget: a rank can legitimately stall
|
||||
# for seconds (layerwise offload, wan2.2 expert-tower swaps), and expiry now
|
||||
# retires the transport on every rank and fails the request
|
||||
@@ -405,6 +413,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Enable cache-dit acceleration for DiT inference
|
||||
# CUDA-IPC transport for 2-rank Ulysses all-to-all (NVLink same-node)
|
||||
"SGLANG_DIFFUSION_IPC_A2A": _lazy_bool("SGLANG_DIFFUSION_IPC_A2A", "true"),
|
||||
"SGLANG_DIFFUSION_IPC_A2A_MULTI": _lazy_bool(
|
||||
"SGLANG_DIFFUSION_IPC_A2A_MULTI", "false"
|
||||
),
|
||||
"SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS": _lazy_int(
|
||||
"SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS", -1
|
||||
),
|
||||
"SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS": _lazy_float(
|
||||
"SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS", 10000.0
|
||||
),
|
||||
|
||||
+505
@@ -0,0 +1,505 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Copy-engine transport for N-rank Ulysses all-to-all on one host.
|
||||
|
||||
The two-rank ``ipc_a2a`` design widened to every peer: each rank maps the
|
||||
peers' staging buffers into its own device context and writes its chunk for a
|
||||
peer straight into that peer's buffer. A contiguous device-to-device copy runs
|
||||
on the copy engine, so the exchange takes no SMs from the kernels around it --
|
||||
NCCL's all-to-all is an SM kernel, and next to a full-occupancy attention it
|
||||
slows that attention by more than the exchange costs.
|
||||
|
||||
Per peer, a sequence counter published after the copy orders the data before
|
||||
the flag, and the receiver spins on that peer's entry. Double-buffered slots
|
||||
alternate per call; a slot is only rewritten two calls later, after a spin on
|
||||
every peer, and each peer publishes that call only after it has consumed the
|
||||
slot in stream order.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import socket
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.kernels.ops.communication.ipc_a2a import load_ipc_a2a_sync
|
||||
from sglang.kernels.ops.diffusion import pack_qkv_destination_major
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
|
||||
_Unsupported,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Group counts tried, in order, when the caller leaves the choice to the
|
||||
# transport: MiniMax-H3's 56 heads give 28, 14 and 7 per rank at Ulysses 2, 4
|
||||
# and 8, and 7 groups divide all three.
|
||||
_AUTO_PIPELINE_GROUPS = (7, 4, 2)
|
||||
|
||||
|
||||
def _member_devices(group, device: int) -> list[int]:
|
||||
"""Local CUDA ordinal of every member of a same-host group."""
|
||||
world_size = dist.get_world_size(group=group)
|
||||
members: list = [None] * world_size
|
||||
dist.all_gather_object(members, (socket.gethostname(), device), group=group)
|
||||
if len({host for host, _ in members}) != 1:
|
||||
raise _Unsupported("requires every Ulysses rank on the same host")
|
||||
devices = [dev for _, dev in members]
|
||||
if len(set(devices)) != world_size:
|
||||
raise _Unsupported(f"Ulysses ranks share CUDA devices {devices}")
|
||||
return devices
|
||||
|
||||
|
||||
class _StreamMemOps:
|
||||
"""Front-end signal and wait (cuStreamWriteValue32 / cuStreamWaitValue32).
|
||||
|
||||
Neither holds an SM, so a wait queued next to a persistent attention kernel
|
||||
costs it nothing (a spin kernel would pin an SM and push an attention CTA
|
||||
into the tail), and a signal queued behind a copy lands when the copy
|
||||
retires instead of when an SM frees up.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
import ctypes
|
||||
|
||||
cuda = ctypes.CDLL("libcuda.so.1")
|
||||
args = [ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint32, ctypes.c_uint]
|
||||
self._write = cuda.cuStreamWriteValue32_v2
|
||||
self._wait = cuda.cuStreamWaitValue32_v2
|
||||
for fn in (self._write, self._wait):
|
||||
fn.argtypes = args
|
||||
fn.restype = ctypes.c_int
|
||||
# drivers can disable stream memory operations; find out here, inside
|
||||
# the group's agreed initialization, rather than mid-forward
|
||||
flag = torch.zeros(1, dtype=torch.int32, device="cuda")
|
||||
stream = torch.cuda.Stream()
|
||||
try:
|
||||
self.write(stream, flag, 1)
|
||||
self.wait(stream, flag, 1)
|
||||
except RuntimeError as e:
|
||||
raise _Unsupported(f"stream memory operations unavailable ({e})") from e
|
||||
stream.synchronize()
|
||||
|
||||
def write(self, stream, flag: torch.Tensor, value: int) -> None:
|
||||
# default flags fence the stream's earlier writes (the copies) first
|
||||
rc = self._write(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
|
||||
if rc:
|
||||
raise RuntimeError(f"cuStreamWriteValue32 failed with CUresult {rc}")
|
||||
|
||||
def wait(self, stream, flag: torch.Tensor, value: int) -> None:
|
||||
# CU_STREAM_WAIT_VALUE_GEQ compares cyclically, so counters may wrap
|
||||
rc = self._wait(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
|
||||
if rc:
|
||||
raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}")
|
||||
|
||||
|
||||
class _PipelineBuffers:
|
||||
"""Receive slots and flags for one pipelined-attention shape.
|
||||
|
||||
Input slots are group-major ([2, groups, world, chunk]) so a group's rows
|
||||
form one [S, group_heads, 3 * head_dim] tensor for attention; output slots
|
||||
are source-major ([2, world, groups, chunk]), each block a peer's rows for
|
||||
one head group, copied into the merged output as it arrives.
|
||||
"""
|
||||
|
||||
def __init__(self, state, s_local, heads, head_dim, groups, dtype):
|
||||
world = state.world
|
||||
self.group_heads = heads // world // groups
|
||||
rows = s_local * self.group_heads
|
||||
zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda")
|
||||
self.inb = state._share(
|
||||
zeros(2, groups, world, rows * 3 * head_dim), state.group
|
||||
)
|
||||
self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group)
|
||||
# fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g
|
||||
# peer p has delivered. Groups finish on separate streams, so a single
|
||||
# output counter could be lowered by a signal that lands late.
|
||||
self.fin = state._share(zeros(world, dt=torch.int32), state.group)
|
||||
self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group)
|
||||
self.send = torch.empty(
|
||||
world * groups,
|
||||
s_local,
|
||||
self.group_heads,
|
||||
3 * head_dim,
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
)
|
||||
self.calls = 0
|
||||
|
||||
|
||||
class IpcA2AMultiState:
|
||||
def __init__(self):
|
||||
self.ops = None
|
||||
# insertion-ordered so eviction is identical on every rank
|
||||
self.staging = OrderedDict()
|
||||
self.flags = None
|
||||
self.peer_flags = None
|
||||
self.seq = None
|
||||
self.timed_out = None
|
||||
self.peer_timed_out = None
|
||||
self.budget_ns = 0
|
||||
self.max_buffers = 0
|
||||
self.calls = 0
|
||||
self.rank = None
|
||||
self.world = 0
|
||||
self.group = None
|
||||
self.failed = False
|
||||
self.inited = False
|
||||
self.memops = None
|
||||
self.pipe_in = None
|
||||
self.pipe_group_streams = []
|
||||
# pipeline shapes whose buffers did not fit on some rank; agreed by all
|
||||
self.declined = set()
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop mappings that belong to a model-parallel group being replaced."""
|
||||
self.__init__()
|
||||
|
||||
def drop_staging(self) -> None:
|
||||
if not self.staging:
|
||||
return
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
self.staging.clear()
|
||||
|
||||
def _share(self, t: torch.Tensor, group) -> list[torch.Tensor]:
|
||||
"""Every member's `t`, peers' re-opened in the LOCAL device context
|
||||
(a mapping is only dereferenceable from the context that opened it)."""
|
||||
from torch.multiprocessing.reductions import reduce_tensor
|
||||
|
||||
handles: list = [None] * self.world
|
||||
dist.all_gather_object(handles, reduce_tensor(t), group=group)
|
||||
dev = torch.cuda.current_device()
|
||||
out = []
|
||||
for member, (fn, args) in enumerate(handles):
|
||||
if member == self.rank:
|
||||
out.append(t)
|
||||
continue
|
||||
args = list(args)
|
||||
for i, v in enumerate(args):
|
||||
if isinstance(v, torch.device):
|
||||
args[i] = torch.device(f"cuda:{dev}")
|
||||
elif isinstance(v, int) and i == 6:
|
||||
# rebuild_cuda_tensor positional device index
|
||||
args[i] = dev
|
||||
out.append(fn(*args))
|
||||
return out
|
||||
|
||||
def init(self, group) -> None:
|
||||
import ctypes
|
||||
|
||||
self.rank = dist.get_rank(group=group)
|
||||
self.world = dist.get_world_size(group=group)
|
||||
self.group = group
|
||||
dev = torch.cuda.current_device()
|
||||
devices = _member_devices(group, dev)
|
||||
peers = [d for member, d in enumerate(devices) if member != self.rank]
|
||||
missing = [d for d in peers if not torch.cuda.can_device_access_peer(dev, d)]
|
||||
memops_error = None
|
||||
try:
|
||||
memops = _StreamMemOps()
|
||||
except _Unsupported as e:
|
||||
memops_error = e
|
||||
# agree before the handle exchange below: a rank that bailed out here
|
||||
# alone would leave the others blocked inside that collective
|
||||
usable = int(not missing and memops_error is None)
|
||||
agreed = torch.tensor([usable], dtype=torch.int32, device="cuda")
|
||||
dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group)
|
||||
if agreed.item() == 0:
|
||||
if missing:
|
||||
raise _Unsupported(
|
||||
f"a Ulysses rank lacks peer-to-peer access (CUDA device {dev} "
|
||||
f"cannot reach {missing})"
|
||||
)
|
||||
raise _Unsupported(
|
||||
str(memops_error) if memops_error else "a Ulysses peer is unsupported"
|
||||
)
|
||||
cudart = ctypes.CDLL("libcudart.so")
|
||||
for peer_dev in peers:
|
||||
# kernel-level dereference of peer mappings needs explicit peer access
|
||||
cudart.cudaDeviceEnablePeerAccess(peer_dev, 0)
|
||||
self.ops = load_ipc_a2a_sync()
|
||||
self.memops = memops
|
||||
# flags[p] is written by peer p; seq[p] counts the calls sent to p
|
||||
self.flags = torch.zeros(self.world, dtype=torch.int32, device="cuda")
|
||||
self.seq = torch.zeros(self.world, dtype=torch.int32, device="cuda")
|
||||
self.timed_out = torch.zeros(1, dtype=torch.int32, device="cuda")
|
||||
self.budget_ns = int(envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS * 1e6)
|
||||
self.max_buffers = envs.SGLANG_DIFFUSION_IPC_A2A_MAX_BUFFERS
|
||||
self.peer_flags = self._share(self.flags, group)
|
||||
self.peer_timed_out = self._share(self.timed_out, group)
|
||||
self.inited = True
|
||||
|
||||
def get_staging(self, chunk: int, dtype: torch.dtype) -> list[torch.Tensor] | None:
|
||||
"""Every member's two-slot `[2, world, chunk]` receive buffer (mine at
|
||||
my rank). Creation is a group collective, so every rank must reach a
|
||||
new key at the same call site; a miss during capture returns None."""
|
||||
key = (chunk, dtype)
|
||||
buffers = self.staging.get(key)
|
||||
if buffers is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
local = torch.zeros(2, self.world, chunk, dtype=dtype, device="cuda")
|
||||
buffers = self._share(local, self.group)
|
||||
self._insert(key, buffers)
|
||||
return buffers
|
||||
|
||||
def _insert(self, key, value) -> None:
|
||||
if len(self.staging) >= self.max_buffers:
|
||||
# queued copies may still target the evicted peer mappings
|
||||
torch.cuda.synchronize()
|
||||
self.staging.popitem(last=False)
|
||||
self.staging[key] = value
|
||||
|
||||
def pipelined_attention(self, q, k, v, attend, groups: int, fill=None):
|
||||
"""Ulysses attention with the head exchange pipelined against it.
|
||||
|
||||
q/k/v are [s_local, heads, head_dim], sequence-sharded with every head.
|
||||
``attend(q, k, v)`` runs attention on [S, heads / (world * groups),
|
||||
head_dim] views of one head group and must treat heads independently,
|
||||
which keeps the result bit-identical to the sequential exchange.
|
||||
Group g's copies run on the copy engine while group g-1 attends, and
|
||||
each group attends on its own stream and sends its output from there,
|
||||
so no stream waits on a kernel still running on another. Returns
|
||||
[s_local, heads, head_dim], or None when this call cannot pipeline.
|
||||
``groups < 0`` picks the count and steps aside, on every rank, when
|
||||
the buffers would not fit.
|
||||
|
||||
``fill(head_start, head_count, dst)``, when given, writes those heads'
|
||||
q/k/v into a [s_local, head_count, 3 * head_dim] block in place of the
|
||||
pack (the caller's QK-norm writes its output there); blocks for this
|
||||
rank go straight into its own receive slot. It is only called once the
|
||||
call is known to pipeline.
|
||||
"""
|
||||
world, r = self.world, self.rank
|
||||
s_local, heads, head_dim = q.shape
|
||||
auto = groups < 0
|
||||
if auto:
|
||||
groups = next(
|
||||
(n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0
|
||||
)
|
||||
if (
|
||||
not groups
|
||||
or heads % (world * groups)
|
||||
or not (q.shape == k.shape == v.shape)
|
||||
):
|
||||
return None
|
||||
key = ("pipeline", s_local, heads, head_dim, groups, q.dtype)
|
||||
bufs = self.staging.get(key)
|
||||
if bufs is None:
|
||||
if torch.cuda.is_current_stream_capturing() or key in self.declined:
|
||||
return None
|
||||
# send + both receive and output slots: 11 * s_local * heads * head_dim
|
||||
need = 11 * s_local * heads * head_dim * q.element_size()
|
||||
if auto and not self._fits_on_every_rank(need):
|
||||
logger.info(
|
||||
"Ulysses pipeline buffers (%.1f GiB) do not fit; using the "
|
||||
"sequential exchange for this shape",
|
||||
need / 2**30,
|
||||
)
|
||||
self.declined.add(key)
|
||||
return None
|
||||
bufs = _PipelineBuffers(self, s_local, heads, head_dim, groups, q.dtype)
|
||||
self._insert(key, bufs)
|
||||
if self.pipe_in is None:
|
||||
self.pipe_in = torch.cuda.Stream()
|
||||
while len(self.pipe_group_streams) < groups:
|
||||
self.pipe_group_streams.append(torch.cuda.Stream())
|
||||
mem, cin, hg = self.memops, self.pipe_in, bufs.group_heads
|
||||
main = torch.cuda.current_stream()
|
||||
slot = bufs.calls % 2
|
||||
base = bufs.calls * groups
|
||||
bufs.calls += 1
|
||||
call = bufs.calls
|
||||
peers = [(r + step) % world for step in range(1, world)]
|
||||
own = []
|
||||
if fill is None:
|
||||
# block p * groups + g is contiguous: group g of the heads rank p owns
|
||||
send = pack_qkv_destination_major(q, k, v, world * groups, out=bufs.send)
|
||||
cin.wait_stream(main)
|
||||
with torch.cuda.stream(cin):
|
||||
for g in range(groups):
|
||||
bufs.inb[r][slot, g, r].copy_(
|
||||
send[r * groups + g].view(-1), non_blocking=True
|
||||
)
|
||||
own.append(cin.record_event())
|
||||
for p in peers:
|
||||
bufs.inb[p][slot, g, r].copy_(
|
||||
send[p * groups + g].view(-1), non_blocking=True
|
||||
)
|
||||
for p in peers:
|
||||
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
|
||||
else:
|
||||
# Group by group: the peers' blocks first, so their copies start
|
||||
# while this rank fills its own block straight into its slot.
|
||||
send = bufs.send
|
||||
block = lambda t: t.view(s_local, hg, 3 * head_dim)
|
||||
sent = []
|
||||
for g in range(groups):
|
||||
for p in peers:
|
||||
fill((p * groups + g) * hg, hg, block(send[p * groups + g]))
|
||||
sent.append(main.record_event())
|
||||
fill((r * groups + g) * hg, hg, block(bufs.inb[r][slot, g, r]))
|
||||
own.append(main.record_event())
|
||||
with torch.cuda.stream(cin):
|
||||
for g in range(groups):
|
||||
cin.wait_event(sent[g])
|
||||
for p in peers:
|
||||
bufs.inb[p][slot, g, r].copy_(
|
||||
send[p * groups + g].view(-1), non_blocking=True
|
||||
)
|
||||
for p in peers:
|
||||
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
|
||||
# Head block b = p * groups + g of the output is group g of rank p's
|
||||
# heads. Each group stream writes its blocks as soon as they exist, so
|
||||
# the merge overlaps the groups still attending instead of trailing them.
|
||||
merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device)
|
||||
merged_blocks = merged.view(s_local, world * groups, hg, head_dim)
|
||||
done = []
|
||||
for g in range(groups):
|
||||
stream = self.pipe_group_streams[g]
|
||||
stream.wait_event(own[g])
|
||||
with torch.cuda.stream(stream):
|
||||
for p in peers:
|
||||
mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1)
|
||||
qkv = bufs.inb[r][slot, g].view(world * s_local, hg, 3 * head_dim)
|
||||
out = attend(
|
||||
qkv[..., :head_dim],
|
||||
qkv[..., head_dim : 2 * head_dim],
|
||||
qkv[..., 2 * head_dim :],
|
||||
)
|
||||
rows = lambda p: out[p * s_local : (p + 1) * s_local]
|
||||
for p in peers:
|
||||
bufs.outb[p][slot, r, g].copy_(
|
||||
rows(p).reshape(-1), non_blocking=True
|
||||
)
|
||||
for p in peers:
|
||||
mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call)
|
||||
merged_blocks[:, r * groups + g].copy_(rows(r))
|
||||
for p in peers:
|
||||
mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call)
|
||||
merged_blocks[:, p * groups + g].copy_(
|
||||
bufs.outb[r][slot, p, g].view(s_local, hg, head_dim)
|
||||
)
|
||||
done.append(stream.record_event())
|
||||
for event in done:
|
||||
main.wait_event(event)
|
||||
# the next call's pack rewrites `send`, which the copy stream reads
|
||||
main.wait_stream(cin)
|
||||
return merged
|
||||
|
||||
def _fits_on_every_rank(self, nbytes: int) -> bool:
|
||||
"""Whether every rank can spare twice `nbytes`, counting the allocator's
|
||||
cached blocks; a group collective, so all ranks reach the same answer."""
|
||||
free, _ = torch.cuda.mem_get_info()
|
||||
cached = torch.cuda.memory_reserved() - torch.cuda.memory_allocated()
|
||||
fits = torch.tensor(
|
||||
[int(free + cached >= 2 * nbytes)], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
dist.all_reduce(fits, op=dist.ReduceOp.MIN, group=self.group)
|
||||
return bool(fits.item())
|
||||
|
||||
def exchange(self, send: torch.Tensor) -> torch.Tensor | None:
|
||||
"""``all_to_all_single`` with equal splits: row p of the contiguous
|
||||
``[world, chunk]`` `send` goes to rank p, and row p of the returned
|
||||
``[world, chunk]`` view is what rank p sent me."""
|
||||
if send.shape[0] != self.world or not send.is_contiguous():
|
||||
return None
|
||||
buffers = self.get_staging(send[0].numel(), send.dtype)
|
||||
if buffers is None:
|
||||
return None
|
||||
slot = self.calls % 2
|
||||
self.calls += 1
|
||||
r = self.rank
|
||||
buffers[r][slot, r].copy_(send[r].view(-1), non_blocking=True)
|
||||
# Step k sends to r + k: every step is a permutation, so each rank
|
||||
# receives from exactly one peer at a time instead of all of them
|
||||
# converging on the same destination.
|
||||
peers = [(r + k) % self.world for k in range(1, self.world)]
|
||||
for p in peers:
|
||||
buffers[p][slot, r].copy_(send[p].view(-1), non_blocking=True)
|
||||
self.ops.bump_signal(
|
||||
self.seq.narrow(0, p, 1), self.peer_flags[p].narrow(0, r, 1)
|
||||
)
|
||||
for p in peers:
|
||||
self.ops.spin_wait(
|
||||
self.flags.narrow(0, p, 1),
|
||||
self.seq.narrow(0, p, 1),
|
||||
self.timed_out,
|
||||
self.peer_timed_out[p],
|
||||
self.budget_ns,
|
||||
)
|
||||
return buffers[r][slot]
|
||||
|
||||
def check_timeout(self) -> None:
|
||||
"""Retire the transport on every rank if any rank's spin expired.
|
||||
|
||||
A spin only flags the peer it waited on, so agree across the whole group
|
||||
at the request boundary: a rank that retired alone would post an NCCL
|
||||
all-to-all that its IPC peers never join. Expiry means a peer went
|
||||
silent for the whole budget and that exchange returned incomplete data,
|
||||
so this raises instead of serving the result.
|
||||
"""
|
||||
if self.failed or not self.inited:
|
||||
return
|
||||
expired = self.timed_out.clone()
|
||||
dist.all_reduce(expired, op=dist.ReduceOp.MAX, group=self.group)
|
||||
if expired.item() == 0:
|
||||
return
|
||||
self.failed = True
|
||||
raise RuntimeError(
|
||||
"IPC all-to-all gave up waiting for a peer after "
|
||||
f"{envs.SGLANG_DIFFUSION_IPC_A2A_TIMEOUT_MS:g} ms, so that exchange "
|
||||
"returned incomplete data. The transport is now disabled on every "
|
||||
"rank; retry the request over NCCL, or set "
|
||||
"SGLANG_DIFFUSION_IPC_A2A_MULTI=0."
|
||||
)
|
||||
|
||||
|
||||
IPC_A2A_MULTI = IpcA2AMultiState()
|
||||
|
||||
|
||||
def ipc_a2a_multi_ready(group, enabled: bool | None = None) -> bool:
|
||||
"""True when the N-rank transport is enabled and initialized on every
|
||||
member of `group` (initializes lazily on the first eager call)."""
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
if not (envs.SGLANG_DIFFUSION_IPC_A2A_MULTI if enabled is None else enabled):
|
||||
return False
|
||||
if IPC_A2A_MULTI.group is not None and IPC_A2A_MULTI.group is not group:
|
||||
IPC_A2A_MULTI.reset()
|
||||
if IPC_A2A_MULTI.failed:
|
||||
return False
|
||||
if IPC_A2A_MULTI.inited:
|
||||
return True
|
||||
if not current_platform.is_cuda() or torch.cuda.is_current_stream_capturing():
|
||||
return False
|
||||
ok = True
|
||||
try:
|
||||
IPC_A2A_MULTI.init(group)
|
||||
except _Unsupported as e:
|
||||
logger.info("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e)
|
||||
ok = False
|
||||
except Exception as e:
|
||||
logger.debug("IPC all-to-all (multi-rank) initialization failed", exc_info=True)
|
||||
logger.warning("IPC all-to-all (multi-rank) unavailable (%s); using NCCL", e)
|
||||
ok = False
|
||||
# every member must take the same path, or an IPC rank waits on an NCCL one
|
||||
agreed = torch.tensor([int(ok)], dtype=torch.int32, device="cuda")
|
||||
dist.all_reduce(agreed, op=dist.ReduceOp.MIN, group=group)
|
||||
if agreed.item() == 0:
|
||||
IPC_A2A_MULTI.reset()
|
||||
IPC_A2A_MULTI.failed = True
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def ulysses_pipelined_attention(q, k, v, attend, groups: int, fill=None):
|
||||
"""Pipelined Ulysses attention over the copy-engine transport, or None
|
||||
when the Ulysses group cannot use it (the caller keeps its NCCL path)."""
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
|
||||
|
||||
group = get_sp_group().ulysses_group
|
||||
if group is None or not ipc_a2a_multi_ready(group, enabled=True):
|
||||
return None
|
||||
return IPC_A2A_MULTI.pipelined_attention(q, k, v, attend, groups, fill=fill)
|
||||
@@ -868,9 +868,11 @@ def destroy_model_parallel() -> None:
|
||||
# The IPC transport keeps CUDA mappings associated with the current
|
||||
# Ulysses group. Drop them before tearing down the process groups.
|
||||
from .device_communicators.ipc_a2a import IPC_A2A
|
||||
from .device_communicators.ipc_a2a_multi import IPC_A2A_MULTI
|
||||
from .parallel_groups import PROCESS_GROUP
|
||||
|
||||
IPC_A2A.reset()
|
||||
IPC_A2A_MULTI.reset()
|
||||
|
||||
destroyed_groups = []
|
||||
for group in (
|
||||
|
||||
@@ -88,6 +88,17 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
|
||||
assert ulysses_pg is not None, "Ulysses process group is not initialized."
|
||||
x_shape = x.shape
|
||||
x = x.flatten().contiguous()
|
||||
if x.is_cuda:
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
ipc_a2a_multi_ready,
|
||||
)
|
||||
|
||||
if ipc_a2a_multi_ready(ulysses_pg):
|
||||
world_size = torch.distributed.get_world_size(group=ulysses_pg)
|
||||
received = IPC_A2A_MULTI.exchange(x.view(world_size, -1))
|
||||
if received is not None:
|
||||
return received.view(x_shape)
|
||||
if role is None:
|
||||
output = torch.empty_like(x)
|
||||
else:
|
||||
|
||||
@@ -37,6 +37,9 @@ from sglang.multimodal_gen.runtime.distributed import (
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
|
||||
IPC_A2A,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_classifier_free_guidance_rank,
|
||||
@@ -559,6 +562,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
# request boundary: the IPC watchdog flag is a device read, illegal
|
||||
# inside a graph capture and too costly per exchange
|
||||
IPC_A2A.check_timeout()
|
||||
IPC_A2A_MULTI.check_timeout()
|
||||
if len(batch) > 1:
|
||||
if return_req:
|
||||
raise ValueError(
|
||||
@@ -1210,6 +1214,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
from sglang.multimodal_gen.runtime.layers.usp import drop_a2a_staging_buffers
|
||||
|
||||
IPC_A2A.drop_staging()
|
||||
IPC_A2A_MULTI.drop_staging()
|
||||
drop_a2a_staging_buffers()
|
||||
torch.get_device_module().empty_cache()
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.kernels.ops.diffusion import (
|
||||
can_use_mxfp8_swizzled,
|
||||
can_use_silu_mul_mxfp8,
|
||||
fused_inplace_qknorm_rope,
|
||||
fused_qknorm_rope_out_of_place,
|
||||
indexed_gate_bf16,
|
||||
indexed_gate_bf16_,
|
||||
indexed_scale_shift_bf16_,
|
||||
@@ -653,6 +654,122 @@ class MiniMaxH3TimeEmbedder(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
def _minimax_h3_bind_attention_backend(
|
||||
attention: MiniMaxH3Attention, dtype: torch.dtype
|
||||
) -> None:
|
||||
if attention._attention_impl is None:
|
||||
attention._set_attention_backend(
|
||||
get_attn_backend(
|
||||
attention.head_dim,
|
||||
dtype,
|
||||
selected_attention_backend=attention._selected_attention_backend,
|
||||
attention_requirements=AttentionRequirements(packed_varlen=True),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _minimax_h3_pipelined_dense_attention(
|
||||
attention: MiniMaxH3Attention,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
*,
|
||||
cu_seqlens: torch.Tensor,
|
||||
cu_seqlens_host: tuple[int, ...] | None,
|
||||
max_seqlen: int,
|
||||
fill=None,
|
||||
) -> torch.Tensor | None:
|
||||
"""Dense FA with the Ulysses exchange pipelined over head groups.
|
||||
|
||||
Returns None (caller keeps the sequential exchange) unless head-group
|
||||
pipelining is enabled and the dense FlashAttention path would run.
|
||||
"""
|
||||
groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS
|
||||
if groups in (0, 1) or torch.compiler.is_compiling():
|
||||
return None
|
||||
if attention._attention_backend_enum is not AttentionBackendEnum.FA:
|
||||
return None
|
||||
impl = attention._attention_impl
|
||||
if impl._request_skip_softmax_threshold()[0]:
|
||||
return None
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
ulysses_pipelined_attention,
|
||||
)
|
||||
|
||||
def attend(q_group, k_group, v_group):
|
||||
return impl.forward_varlen(
|
||||
q_group,
|
||||
k_group,
|
||||
v_group,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
)
|
||||
|
||||
return ulysses_pipelined_attention(q, k, v, attend, groups, fill=fill)
|
||||
|
||||
|
||||
def _minimax_h3_qknorm_rope_pipelined_attention(
|
||||
attention: MiniMaxH3Attention,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
rope_cache: tuple[torch.Tensor, torch.Tensor],
|
||||
*,
|
||||
cu_seqlens: torch.Tensor,
|
||||
cu_seqlens_host: tuple[int, ...] | None,
|
||||
max_seqlen: int,
|
||||
) -> torch.Tensor | None:
|
||||
"""The pipelined attention with QK-norm + RoPE writing into the exchange.
|
||||
|
||||
q/k are the raw projections. Each destination block gets its heads'
|
||||
normalized q/k and v written in place, so the in-place norm and the pack
|
||||
become one pass. Same kernel arithmetic, so the result is bit-identical.
|
||||
Returns None, having written nothing, when the call cannot pipeline.
|
||||
"""
|
||||
# under graph capture the forward keeps the in-place norm, and the pipeline
|
||||
# runs from the attention core's eager break point instead
|
||||
if (
|
||||
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS in (0, 1)
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
return None
|
||||
_minimax_h3_bind_attention_backend(attention, q.dtype)
|
||||
cos_sin_cache, positions = rope_cache
|
||||
head_dim = attention.head_dim
|
||||
q_weight, k_weight = attention.q_norm.weight, attention.k_norm.weight
|
||||
|
||||
def fill(head_start: int, head_count: int, dst: torch.Tensor) -> None:
|
||||
heads = slice(head_start, head_start + head_count)
|
||||
fused_qknorm_rope_out_of_place(
|
||||
q[:, heads],
|
||||
k[:, heads],
|
||||
dst[..., :head_dim],
|
||||
dst[..., head_dim : 2 * head_dim],
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
is_neox=True,
|
||||
eps=attention.q_norm.eps,
|
||||
head_dim=head_dim,
|
||||
rope_dim=cos_sin_cache.shape[-1],
|
||||
round_norm_before_rope=True,
|
||||
)
|
||||
dst[..., 2 * head_dim :].copy_(v[:, heads])
|
||||
|
||||
return _minimax_h3_pipelined_dense_attention(
|
||||
attention,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
max_seqlen=max_seqlen,
|
||||
fill=fill,
|
||||
)
|
||||
|
||||
|
||||
def _minimax_h3_attention_core_impl(
|
||||
attention: MiniMaxH3Attention,
|
||||
q: torch.Tensor,
|
||||
@@ -674,6 +791,21 @@ def _minimax_h3_attention_core_impl(
|
||||
kernel and sequence-parallel collectives execute eagerly.
|
||||
"""
|
||||
|
||||
_minimax_h3_bind_attention_backend(attention, q.dtype)
|
||||
|
||||
if ulysses_active and not ring_active and gate_compress is None:
|
||||
out = _minimax_h3_pipelined_dense_attention(
|
||||
attention,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
if out is not None:
|
||||
return out
|
||||
|
||||
if ulysses_active:
|
||||
from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_all_gather,
|
||||
@@ -683,16 +815,6 @@ def _minimax_h3_attention_core_impl(
|
||||
|
||||
q, k, v = _usp_input_all_to_all_packed_qkv(q, k, v)
|
||||
|
||||
if attention._attention_impl is None:
|
||||
attention._set_attention_backend(
|
||||
get_attn_backend(
|
||||
attention.head_dim,
|
||||
q.dtype,
|
||||
selected_attention_backend=attention._selected_attention_backend,
|
||||
attention_requirements=AttentionRequirements(packed_varlen=True),
|
||||
)
|
||||
)
|
||||
|
||||
if attention._attention_backend_enum is AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3:
|
||||
attn_metadata = (
|
||||
get_forward_context().attn_metadata
|
||||
@@ -1159,6 +1281,22 @@ class MiniMaxH3Attention(nn.Module):
|
||||
else:
|
||||
cos_sin_cache, positions = rope_cache
|
||||
if self._use_fused_qknorm_rope and not torch.compiler.is_compiling():
|
||||
if ulysses_active and not ring_active:
|
||||
out = _minimax_h3_qknorm_rope_pipelined_attention(
|
||||
self,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
rope_cache,
|
||||
cu_seqlens=cu_seqlens,
|
||||
cu_seqlens_host=cu_seqlens_host,
|
||||
max_seqlen=max_seqlen,
|
||||
)
|
||||
if out is not None:
|
||||
out, _ = self.out_proj(
|
||||
out.reshape(total, self.num_heads * self.head_dim)
|
||||
)
|
||||
return out
|
||||
fused_inplace_qknorm_rope(
|
||||
q,
|
||||
k,
|
||||
|
||||
@@ -1561,6 +1561,7 @@ STANDALONE_FILES = {
|
||||
"../single_test_file/test_disagg_server.py",
|
||||
"../single_test_file/test_ar_models.py",
|
||||
"../single_test_file/test_ipc_a2a_2_gpu.py",
|
||||
"../single_test_file/test_ipc_a2a_multi_gpu.py",
|
||||
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py",
|
||||
"../single_test_file/test_encoder_fold_srt_2_gpu.py",
|
||||
"../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py",
|
||||
@@ -1602,6 +1603,8 @@ STANDALONE_FILE_EST_TIMES = {
|
||||
"../single_test_file/test_ar_models.py": 600.0,
|
||||
# no model load; the cost is the one-time JIT build of the sync kernels
|
||||
"../single_test_file/test_ipc_a2a_2_gpu.py": 240.0,
|
||||
# 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT
|
||||
"../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0,
|
||||
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0,
|
||||
"../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0,
|
||||
# ~60 s locally with a warm HF cache (load + one capture + 4 steps);
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
"""The copy-engine all-to-all and the pipelined Ulysses attention must match the
|
||||
sequential exchange bit for bit.
|
||||
|
||||
Both only activate with SGLANG_DIFFUSION_IPC_A2A_MULTI / a pipeline group count
|
||||
on a same-host group with peer-to-peer access, so nothing in the single-GPU
|
||||
suite reaches them. Runs on 2 ranks, and on 4 when four GPUs are visible:
|
||||
|
||||
pytest -v python/sglang/multimodal_gen/test/single_test_file/test_ipc_a2a_multi_gpu.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _attend(q, k, v):
|
||||
# head-independent and exact under any head grouping; the flips and rolls
|
||||
# read across the whole gathered sequence, so a misplaced shard shows up
|
||||
return (q.float() * 2 + k.flip(0).float() - v.roll(1, 0).float()).to(q.dtype)
|
||||
|
||||
|
||||
def _worker() -> int:
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
|
||||
rank = int(os.environ["RANK"])
|
||||
world = int(os.environ["WORLD_SIZE"])
|
||||
torch.cuda.set_device(rank)
|
||||
maybe_init_distributed_environment_and_model_parallel(
|
||||
tp_size=1, sp_size=world, ulysses_degree=world
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
ulysses_pipelined_attention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_all_to_all_single,
|
||||
_usp_input_all_to_all_packed_qkv,
|
||||
_usp_output_all_to_all,
|
||||
)
|
||||
|
||||
peers = [d for d in range(world) if d != rank]
|
||||
if not all(torch.cuda.can_device_access_peer(rank, d) for d in peers):
|
||||
print("SKIP no peer-to-peer access between the devices", flush=True)
|
||||
return 0
|
||||
|
||||
failures = []
|
||||
|
||||
def sequential(q, k, v):
|
||||
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
|
||||
qs, ks, vs = _usp_input_all_to_all_packed_qkv(q, k, v)
|
||||
return _usp_output_all_to_all(_attend(qs, ks, vs)[None], head_dim=2)[0]
|
||||
|
||||
# (s_local, heads, head_dim, groups); heads % (world * groups) == 0
|
||||
cases = [(96, 16, 64, 2), (96, 16, 64, 4), (40, 32, 128, 2)]
|
||||
if world == 2:
|
||||
cases.append((128, 16, 64, 8))
|
||||
for s_local, heads, head_dim, groups in cases:
|
||||
# three calls in a row: the slots alternate and the counters advance
|
||||
for call in range(3):
|
||||
torch.manual_seed(
|
||||
1000 * call + s_local
|
||||
) # identical full tensors on every rank
|
||||
full = [
|
||||
torch.randn(
|
||||
s_local * world,
|
||||
heads,
|
||||
head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full)
|
||||
ref = sequential(q, k, v)
|
||||
got = ulysses_pipelined_attention(q, k, v, _attend, groups)
|
||||
if got is None:
|
||||
failures.append(
|
||||
f"pipelined returned None for {(s_local, heads, head_dim, groups)}"
|
||||
)
|
||||
elif not torch.equal(ref, got):
|
||||
failures.append(
|
||||
f"pipelined {(s_local, heads, head_dim, groups)} call {call}"
|
||||
)
|
||||
|
||||
# fill: the caller writes each destination block itself (H3's QK-norm does),
|
||||
# own blocks straight into this rank's receive slot
|
||||
def transform(t, scale):
|
||||
return (t.float() * scale + 1).to(t.dtype)
|
||||
|
||||
for s_local, heads, head_dim, groups in cases:
|
||||
for call in range(3):
|
||||
torch.manual_seed(2000 + 1000 * call + s_local)
|
||||
full = [
|
||||
torch.randn(
|
||||
s_local * world,
|
||||
heads,
|
||||
head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full)
|
||||
|
||||
def fill(head_start, head_count, dst):
|
||||
h = slice(head_start, head_start + head_count)
|
||||
dst[..., :head_dim].copy_(transform(q[:, h], 2))
|
||||
dst[..., head_dim : 2 * head_dim].copy_(transform(k[:, h], 3))
|
||||
dst[..., 2 * head_dim :].copy_(v[:, h])
|
||||
|
||||
ref = sequential(transform(q, 2), transform(k, 3), v)
|
||||
got = ulysses_pipelined_attention(q, k, v, _attend, groups, fill=fill)
|
||||
if got is None:
|
||||
failures.append(
|
||||
f"filled pipeline returned None for {(s_local, heads, head_dim, groups)}"
|
||||
)
|
||||
elif not torch.equal(ref, got):
|
||||
failures.append(
|
||||
f"filled pipeline {(s_local, heads, head_dim, groups)} call {call}"
|
||||
)
|
||||
|
||||
# groups=-1 picks a count that divides the heads per rank (4 for 16 heads at
|
||||
# 2 ranks, 2 at 4) and still matches
|
||||
torch.manual_seed(77)
|
||||
full = [
|
||||
torch.randn(64 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(3)
|
||||
]
|
||||
q, k, v = (t.narrow(0, rank * 64, 64).contiguous() for t in full)
|
||||
got = ulysses_pipelined_attention(q, k, v, _attend, -1)
|
||||
if got is None or not torch.equal(sequential(q, k, v), got):
|
||||
failures.append("auto group count")
|
||||
|
||||
# a head count the groups cannot split must decline, not mis-shard
|
||||
q = torch.randn(32, 12, 64, dtype=torch.bfloat16, device="cuda")
|
||||
if ulysses_pipelined_attention(q, q, q, _attend, 4) is not None:
|
||||
failures.append("pipelined accepted 12 heads in 4 groups")
|
||||
|
||||
# the plain N-rank exchange behind _usp_all_to_all_single
|
||||
for numel in (world * 1024, world * 4096 * 33):
|
||||
x = torch.randn(numel, dtype=torch.bfloat16, device="cuda")
|
||||
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
|
||||
ref = _usp_all_to_all_single(x.clone())
|
||||
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = True
|
||||
calls = IPC_A2A_MULTI.calls
|
||||
got = _usp_all_to_all_single(x.clone())
|
||||
envs.SGLANG_DIFFUSION_IPC_A2A_MULTI = False
|
||||
if IPC_A2A_MULTI.calls == calls:
|
||||
failures.append(f"exchange of {numel} elements never took the copy engine")
|
||||
elif not torch.equal(ref, got):
|
||||
failures.append(f"exchange of {numel} elements")
|
||||
|
||||
# a comparison where every arm quietly fell back would pass while testing nothing
|
||||
if not IPC_A2A_MULTI.inited or IPC_A2A_MULTI.failed or not IPC_A2A_MULTI.staging:
|
||||
failures.append(
|
||||
f"transport never engaged (inited={IPC_A2A_MULTI.inited} "
|
||||
f"failed={IPC_A2A_MULTI.failed} staged={len(IPC_A2A_MULTI.staging)})"
|
||||
)
|
||||
|
||||
verdict = torch.tensor([len(failures)], device="cuda")
|
||||
dist.all_reduce(verdict)
|
||||
if failures:
|
||||
print(f"rank{rank} MISMATCH: {failures}", flush=True)
|
||||
if rank == 0:
|
||||
print(
|
||||
f"IPC_A2A_MULTI_PARITY {'FAIL' if verdict.item() else 'PASS'} world={world}",
|
||||
flush=True,
|
||||
)
|
||||
dist.barrier()
|
||||
dist.destroy_process_group()
|
||||
return 1 if verdict.item() else 0
|
||||
|
||||
|
||||
class TestIpcA2AMultiGpu(CustomTestCase):
|
||||
def _run(self, world: int, port: int):
|
||||
if not current_platform.is_cuda():
|
||||
self.skipTest("CUDA-IPC transport is unavailable on this platform")
|
||||
if torch.cuda.device_count() < world:
|
||||
self.skipTest(f"needs {world} GPUs")
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"torch.distributed.run",
|
||||
f"--nproc-per-node={world}",
|
||||
f"--master-port={port}",
|
||||
__file__,
|
||||
"--worker",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=1200,
|
||||
)
|
||||
print(proc.stdout[-4000:])
|
||||
if proc.returncode != 0:
|
||||
print(proc.stderr[-4000:], file=sys.stderr)
|
||||
self.assertEqual(proc.returncode, 0, "copy-engine all-to-all diverged")
|
||||
if "SKIP" not in proc.stdout:
|
||||
self.assertIn(f"IPC_A2A_MULTI_PARITY PASS world={world}", proc.stdout)
|
||||
|
||||
def test_two_ranks_match_sequential_bitwise(self):
|
||||
self._run(2, 29527)
|
||||
|
||||
def test_four_ranks_match_sequential_bitwise(self):
|
||||
self._run(4, 29537)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--worker" in sys.argv:
|
||||
raise SystemExit(_worker())
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user