[diffusion] perf: pipeline every USPAttention Ulysses path's exchange over the copy engine (#43521)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-10 21:34:18 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 416ebe6090
commit 72657f7ca5
10 changed files with 1391 additions and 371 deletions
+5 -4
View File
@@ -68,10 +68,11 @@ if TYPE_CHECKING:
# copy-engine all-to-all for Ulysses groups of any size on one host; off by
# default while it is validated. Falls back to NCCL when unavailable.
SGLANG_DIFFUSION_IPC_A2A_MULTI: bool = False
# head groups for pipelining MiniMax-H3's Ulysses exchange against dense
# attention over the copy-engine transport: -1 picks a count that divides
# the heads per rank and steps aside when its buffers do not fit, 0 or 1
# keeps the sequential exchange, N >= 2 forces N groups
# head groups for pipelining the Ulysses exchange against dense attention
# over the copy-engine transport (MiniMax-H3, and USPAttention's plain path):
# -1 picks the most groups that divide the heads per rank while each group's
# call still fills the GPU, and steps aside when the buffers do not fit; 0 or
# 1 keeps the sequential exchange; N >= 2 forces N groups
SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS: int = -1
# a deadlock backstop, not a per-step budget: a rank can legitimately stall
# for seconds (layerwise offload, wan2.2 expert-tower swaps), and expiry now
@@ -9,15 +9,19 @@ NCCL's all-to-all is an SM kernel, and next to a full-occupancy attention it
slows that attention by more than the exchange costs.
Per peer, a sequence counter published after the copy orders the data before
the flag, and the receiver spins on that peer's entry. Double-buffered slots
alternate per call; a slot is only rewritten two calls later, after a spin on
every peer, and each peer publishes that call only after it has consumed the
slot in stream order.
the flag, and the receiver spins on that peer's entry. The plain exchange's
double-buffered slots alternate per call; a slot is only rewritten two calls
later, after a spin on every peer, and each peer publishes that call only after
it has consumed the slot in stream order. The pipelined attention needs one set
of slots (see ``_PipelinePool``).
"""
import dataclasses
import functools
import logging
import socket
from collections import OrderedDict
from dataclasses import dataclass
import torch
import torch.distributed as dist
@@ -32,8 +36,86 @@ logger = logging.getLogger(__name__)
# Group counts tried, in order, when the caller leaves the choice to the
# transport: MiniMax-H3's 56 heads give 28, 14 and 7 per rank at Ulysses 2, 4
# and 8, and 7 groups divide all three.
_AUTO_PIPELINE_GROUPS = (7, 4, 2)
# and 8, and 7 groups divide all three; Wan's 40 and 12 heads need 5 and 3.
_AUTO_PIPELINE_GROUPS = (7, 5, 4, 3, 2)
# Each group's attention call must fill this many waves of 128-row tiles: below
# it the call's tail costs more than the overlap saves (cuDNN SDPA on B300 runs
# Wan 1.3B at Ulysses 4 in three one-head groups 44% slower than sequentially).
_MIN_GROUP_WAVES = 4
@functools.lru_cache(maxsize=None)
def _sm_count(device: int) -> int:
return torch.cuda.get_device_properties(device).multi_processor_count
@dataclass(frozen=True)
class _PipelinePlan:
"""Rows and heads of one pipelined attention call.
Each part (q, k, v) is a ``[batch, s_local, heads, head_dim]`` sequence
shard plus ``replicated`` rows that every rank holds and that stay out of
the exchange (text tokens), attended before the gathered sequence when
``replicated_first`` and after it otherwise. A receive slot holds one part
of one head group as attention reads it: its replicated rows and every
rank's shard in rank order. The output keeps q's rows in this rank's
layout: its replicated rows and this rank's shard.
"""
batch: int
s_local: int
heads: tuple[int, int, int]
head_dim: int
replicated: tuple[int, int, int]
replicated_first: bool
world: int
groups: int
def group_heads(self, part: int) -> int:
return self.heads[part] // (self.world * self.groups)
def rows(self, part: int) -> int:
return self.replicated[part] + self.world * self.s_local
def shard_row(self, part: int, rank: int) -> int:
return (self.replicated[part] if self.replicated_first else 0) + (
rank * self.s_local
)
def replicated_row(self, part: int) -> int:
return 0 if self.replicated_first else self.world * self.s_local
@property
def out_rows(self) -> int:
return self.replicated[0] + self.s_local
@property
def out_shard_row(self) -> int:
return self.replicated[0] if self.replicated_first else 0
@property
def out_replicated_row(self) -> int:
return 0 if self.replicated_first else self.s_local
def slot_numel(self, part: int) -> int:
return self.batch * self.rows(part) * self.group_heads(part) * self.head_dim
def out_block_numel(self) -> int:
return self.batch * self.out_rows * self.group_heads(0) * self.head_dim
def _auto_pipeline_groups(plan: _PipelinePlan) -> int:
"""The most head groups whose calls still fill the GPU, or 0 for none."""
tiles = plan.batch * -(-plan.rows(0) // 128)
floor = _MIN_GROUP_WAVES * _sm_count(torch.cuda.current_device())
for n in _AUTO_PIPELINE_GROUPS:
split = plan.world * n
if (
all(h % split == 0 for h in plan.heads)
and plan.heads[0] // split * tiles >= floor
):
return n
return 0
def _member_devices(group, device: int) -> list[int]:
@@ -114,15 +196,17 @@ class _StreamMemOps:
raise _Unsupported(f"stream memory operations unavailable ({e})") from e
stream.synchronize()
def write(self, stream, flag: torch.Tensor, value: int) -> None:
def write(self, stream, flag: torch.Tensor | int, value: int) -> None:
# default flags fence the stream's earlier writes (the copies) first
rc = self._write(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
ptr = flag if isinstance(flag, int) else flag.data_ptr()
rc = self._write(stream.cuda_stream, ptr, value & 0xFFFFFFFF, 0)
if rc:
raise RuntimeError(f"cuStreamWriteValue32 failed with CUresult {rc}")
def wait(self, stream, flag: torch.Tensor, value: int) -> None:
def wait(self, stream, flag: torch.Tensor | int, value: int) -> None:
# CU_STREAM_WAIT_VALUE_GEQ compares cyclically, so counters may wrap
rc = self._wait(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
ptr = flag if isinstance(flag, int) else flag.data_ptr()
rc = self._wait(stream.cuda_stream, ptr, value & 0xFFFFFFFF, 0)
if rc:
raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}")
@@ -153,41 +237,104 @@ class _StreamMemOps:
raise RuntimeError(f"cuMemcpy2DAsync failed with CUresult {rc}")
class _PipelineBuffers:
"""Receive slots and flags for one pipelined-attention shape.
class _PipelinePool:
"""Receive slots, output blocks and flags that every pipelined call of one
dtype shares, grown to the largest call seen rather than kept per shape.
Input slots are [2, groups, 3, world, chunk]: group g's q (k, v) rows from
every rank form one contiguous [S, group_heads, head_dim] tensor for
attention. Output slots are source-major ([2, world, groups, chunk]), each
block a peer's rows for one head group, copied into the merged output as it
arrives.
One set serves every call, with no slot per call parity. Rank r sends a
group's output blocks only after attending over its slot, and peer p
returns from a call only after merging all of r's blocks; p writes r's
slots for the next call only after that, so after r has read them. And p
writes its next block into r's output block only after attending over r's
rows of that call, which r sends only after its own previous call
returned, so after r merged p's previous block.
"""
def __init__(self, state, s_local, heads, head_dim, groups, dtype):
def __init__(self, state, dtype, in_numel: int, out_numel: int, groups: int):
world = state.world
self.group_heads = heads // world // groups
rows = s_local * self.group_heads
zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda")
self.inb = state._share(
zeros(2, groups, 3, world, rows * head_dim), state.group
)
self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group)
self.sizes = (in_numel, out_numel, groups)
zeros = lambda n, dt=dtype: torch.zeros(n, dtype=dt, device="cuda")
self.inb = state._share(zeros(in_numel), state.group)
self.outb = state._share(zeros(out_numel), state.group)
# fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g
# peer p has delivered. Groups finish on separate streams, so a single
# output counter could be lowered by a signal that lands late.
self.fin = state._share(zeros(world, dt=torch.int32), state.group)
self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group)
# the caller's q/k for each peer block; v moves straight from its source
self.send = torch.empty(
world * groups,
2,
s_local,
self.group_heads,
head_dim,
dtype=dtype,
device="cuda",
)
self.fout = state._share(zeros(groups * world, dt=torch.int32), state.group)
self.inb_ptr = [t.data_ptr() for t in self.inb]
self.outb_ptr = [t.data_ptr() for t in self.outb]
self.fin_ptr = [t.data_ptr() for t in self.fin]
self.fout_ptr = [t.data_ptr() for t in self.fout]
self.calls = 0
self.groups_sent = 0
# the caller's q/k for each peer block when a fill writes them; v moves
# straight from its source
self.send = None
def nbytes(self, esz: int) -> int:
return (self.sizes[0] + self.sizes[1]) * esz
class _PipelineSlots:
"""Where one plan's receive slots and output blocks sit in the pool.
Receive slots, per head group and part: one contiguous ``[batch, rows,
group_heads, head_dim]`` tensor laid out as attention reads it (see
``_PipelinePlan``). Output blocks are source-major: block ``(source,
group)`` is that rank's output rows for this rank, one ``[batch, out_rows,
group_heads, head_dim]`` block in the output's row order, copied into the
merged output as it arrives.
"""
def __init__(self, plan: _PipelinePlan, dtype):
self.plan = plan
self.dtype = dtype
numel = [plan.slot_numel(part) for part in range(3)]
self.part_numel = numel
self.part_offset = (0, numel[0], numel[0] + numel[1])
self.group_numel = sum(numel)
self.out_numel = plan.out_block_numel()
self.in_total = plan.groups * self.group_numel
self.out_total = plan.world * plan.groups * self.out_numel
# raw addresses: the copies are issued per call from plain integers,
# since a tensor view per copy costs more host time than the copy
self.esz = torch.empty((), dtype=dtype).element_size()
self.rows = [plan.rows(part) for part in range(3)]
self.row_bytes = [
plan.group_heads(part) * plan.head_dim * self.esz for part in range(3)
]
self.send_numel = 2 * plan.s_local * plan.group_heads(0) * plan.head_dim
def slot_ptr(self, pool, member, group, part, batch_row, row) -> int:
elems = group * self.group_numel + self.part_offset[part]
return (
pool.inb_ptr[member]
+ elems * self.esz
+ (batch_row * self.rows[part] + row) * self.row_bytes[part]
)
def out_block_ptr(self, pool, member, source, group) -> int:
block = (source * self.plan.groups + group) * self.out_numel
return pool.outb_ptr[member] + block * self.esz
def slot(self, pool, member: int, group: int, part: int) -> torch.Tensor:
"""``member``'s receive slot, mapped in this rank's context."""
plan = self.plan
start = group * self.group_numel + self.part_offset[part]
return (
pool.inb[member]
.narrow(0, start, self.part_numel[part])
.view(plan.batch, plan.rows(part), plan.group_heads(part), plan.head_dim)
)
def send_block(self, pool, peer: int, group: int) -> torch.Tensor:
plan = self.plan
need = plan.world * plan.groups * self.send_numel
if pool.send is None or pool.send.numel() < need:
pool.send = torch.empty(need, dtype=self.dtype, device="cuda")
return pool.send.narrow(
0, (peer * plan.groups + group) * self.send_numel, self.send_numel
).view(2, plan.s_local, plan.group_heads(0), plan.head_dim)
class IpcA2AMultiState:
@@ -213,17 +360,29 @@ class IpcA2AMultiState:
self.pipe_group_streams = []
# pipeline shapes whose buffers did not fit on some rank; agreed by all
self.declined = set()
# (shape, caller signature) checked against the sequential exchange,
# by outcome; agreed by all
self.verified = set()
self.mismatched = set()
# call shapes -> the plan they resolve to, None when they cannot
# pipeline; repeated calls skip straight to the answer
self.plans = {}
# dtype -> the _PipelinePool every pipelined call of that dtype uses
self.pools = {}
# ("pipeline", plan, dtype) -> its _PipelineSlots
self.slots = {}
def reset(self) -> None:
"""Drop mappings that belong to a model-parallel group being replaced."""
self.__init__()
def drop_staging(self) -> None:
if not self.staging:
if not self.staging and not self.pools:
return
if torch.cuda.is_available():
torch.cuda.synchronize()
self.staging.clear()
self.pools.clear()
def _share(self, t: torch.Tensor, group) -> list[torch.Tensor]:
"""Every member's `t`, peers' re-opened in the LOCAL device context
@@ -314,53 +473,112 @@ class IpcA2AMultiState:
self.staging.popitem(last=False)
self.staging[key] = value
def pipelined_attention(self, q, k, v, attend, groups: int, fill=None):
def pipelined_attention(
self,
q,
k,
v,
attend,
groups: int,
fill=None,
sequential=None,
signature=None,
replicated=(None, None, None),
replicated_first: bool = True,
):
"""Ulysses attention with the head exchange pipelined against it.
q/k/v are [s_local, heads, head_dim], sequence-sharded with every head.
``attend(q, k, v)`` runs attention on [S, heads / (world * groups),
head_dim] tensors of one head group and must treat heads independently,
which keeps the result bit-identical to the sequential exchange.
Group g's copies run on the copy engine while group g-1 attends, and
each group attends on its own stream and sends its output from there,
so no stream waits on a kernel still running on another. Every copy,
the merge into the output included, is a pitched copy-engine copy, so
the exchange takes no SMs. Returns [s_local, heads, head_dim], or None
when this call cannot pipeline. ``groups < 0`` picks the count and
steps aside, on every rank, when the buffers would not fit.
q/k/v are ``[batch, s_local, heads, head_dim]`` sequence shards with
every head (``[s_local, heads, head_dim]`` reads as batch 1 and returns
that shape); k and v may have fewer heads than q (GQA).
``replicated`` optionally holds, per part, ``[batch, n, heads,
head_dim]`` rows every rank holds (text tokens); they stay out of the
exchange, attend before the gathered sequence when
``replicated_first`` and after it otherwise, and their output, all
heads, leads (or trails) the returned rows.
``fill(head_start, head_count, q_dst, k_dst)``, when given, writes those
heads' q and k into two [s_local, head_count, head_dim] blocks in place
of a copy (the caller's QK-norm writes its output there); blocks for
this rank go straight into its own receive slot, and v always moves
straight from ``v``. It is only called once the call is known to
pipeline.
``attend(q, k, v)`` runs attention on one head group, ``[batch, rows,
heads / (world * groups), head_dim]`` per part in the sequence order
above, and must treat heads independently, which keeps the result
bit-identical to the sequential exchange. Group g's copies run on the
copy engine while group g-1 attends, and each group attends on its own
stream and sends its output from there, so no stream waits on a kernel
still running on another. Every copy, the merge into the output
included, is a pitched copy-engine copy, so the exchange takes no SMs.
Returns the output rows of this rank, or None when this call cannot
pipeline. ``groups < 0`` picks the count and steps aside, on every
rank, when the buffers would not fit.
``fill(head_start, head_count, q_dst, k_dst)``, when given (batch 1,
no replicated rows, as many k as q heads), writes those heads' q and k
into two [s_local, head_count, head_dim] blocks in place of a copy
(the caller's QK-norm writes its output there); blocks for this rank
go straight into its own receive slot, and v always moves straight
from ``v``. It is only called once the call is known to pipeline.
``sequential()``, when given, returns what the sequential exchange
gives for this call. The first call per plan and ``signature`` (the
caller's attention setup) runs both and keeps the pipeline for them
only if every rank sees the same bytes, so a backend that does not
treat a head group as it treats those heads in the full call never
changes a result. It runs the sequential exchange first, with the
pool released, so that call never holds both at once.
"""
world, r = self.world, self.rank
s_local, heads, head_dim = q.shape
squeeze = q.dim() == 3
if squeeze:
q, k, v = q[None], k[None], v[None]
parts = (q, k, v)
replicated = tuple(replicated)
batch, s_local, _, head_dim = q.shape
auto = groups < 0
if auto:
groups = next(
(n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0
shapes = (
q.shape,
k.shape,
v.shape,
tuple(None if t is None else t.shape for t in replicated),
replicated_first,
groups,
fill is not None,
)
if shapes not in self.plans:
self.plans[shapes] = self._plan(
q, k, v, replicated, replicated_first, groups, fill is not None
)
plan = self.plans[shapes]
if plan is None:
return None
groups = plan.groups
# each token's heads of one group must be one contiguous run to move
# with a pitched copy
copied = (v,) if fill is not None else (q, k, v)
if (
not groups
or heads % (world * groups)
or not (q.shape == k.shape == v.shape)
or any(t.stride(2) != 1 or t.stride(1) != head_dim for t in copied)
):
copied = [v] if fill is not None else [q, k, v]
copied += [t for t in replicated if t is not None]
if not all(t.stride(-1) == 1 and t.stride(-2) == head_dim for t in copied):
return None
key = ("pipeline", s_local, heads, head_dim, groups, q.dtype)
bufs = self.staging.get(key)
if bufs is None:
key = ("pipeline", plan, q.dtype)
check = None if sequential is None else (key, signature)
if check in self.mismatched:
return None
if check in self.verified:
check = None
slots = self.slots.get(key)
if slots is None:
slots = self.slots[key] = _PipelineSlots(plan, q.dtype)
pool = self.pools.get(q.dtype)
sizes = (slots.in_total, slots.out_total, groups)
if pool is not None:
sizes = tuple(map(max, sizes, pool.sizes))
reference = None
if pool is None or pool.sizes != sizes or check is not None:
if torch.cuda.is_current_stream_capturing() or key in self.declined:
return None
# send + both receive and output slots: 10 * s_local * heads * head_dim
need = 10 * s_local * heads * head_dim * q.element_size()
if auto and not self._fits_on_every_rank(need):
esz = q.element_size()
held = 0 if pool is None else pool.nbytes(esz)
need = (sizes[0] + sizes[1]) * esz
if fill is not None:
need += plan.world * groups * slots.send_numel * esz
if auto and not self._fits_on_every_rank(need, reclaimable=held):
logger.info(
"Ulysses pipeline buffers (%.1f GiB) do not fit; using the "
"sequential exchange for this shape",
@@ -368,82 +586,137 @@ class IpcA2AMultiState:
)
self.declined.add(key)
return None
bufs = _PipelineBuffers(self, s_local, heads, head_dim, groups, q.dtype)
self._insert(key, bufs)
if pool is not None:
# queued copies may still target the old mappings
torch.cuda.synchronize()
del self.pools[q.dtype]
pool = None
if check is not None:
reference = sequential()
pool = self.pools[q.dtype] = _PipelinePool(self, q.dtype, *sizes)
if self.pipe_in is None:
self.pipe_in = torch.cuda.Stream()
while len(self.pipe_group_streams) < groups:
self.pipe_group_streams.append(torch.cuda.Stream())
mem, cin, hg = self.memops, self.pipe_in, bufs.group_heads
mem, cin = self.memops, self.pipe_in
main = torch.cuda.current_stream()
slot = bufs.calls % 2
base = bufs.calls * groups
bufs.calls += 1
call = bufs.calls
base = pool.groups_sent
pool.groups_sent += groups
pool.calls += 1
call = pool.calls
peers = [(r + step) % world for step in range(1, world)]
esz = q.element_size()
width = hg * head_dim * esz # one token's heads of one group
chunk = s_local * width # one rank's rows of one group, one of q/k/v
start = lambda p, g: (p * groups + g) * hg # first head of block (p, g)
slot_of = lambda p, g, part: bufs.inb[p][slot, g, part, r] # my rows there
head_bytes = head_dim * esz
row_bytes = slots.row_bytes
# first head of block (p, g) of a part
start = lambda p, g, part: (p * groups + g) * plan.group_heads(part)
# (address, batch pitch, token pitch) of each part's shard and replicated rows
src = [(t.data_ptr(), t.stride(0) * esz, t.stride(1) * esz) for t in parts]
rep_src = [
None if t is None else (t.data_ptr(), t.stride(0) * esz, t.stride(1) * esz)
for t in replicated
]
my_shard_row = [plan.shard_row(part, r) for part in range(3)]
def send_rows(stream, t, p, g, part):
src = t[:, start(p, g)]
mem.copy2d(
stream,
slot_of(p, g, part).data_ptr(),
width,
src.data_ptr(),
t.stride(0) * esz,
width,
s_local,
)
def send_rows(stream, part, p, g):
# my shard of one part and head group into rank p's receive slot
ptr, batch_pitch, token_pitch = src[part]
ptr += start(p, g, part) * head_bytes
for b in range(batch):
mem.copy2d(
stream,
slots.slot_ptr(pool, p, g, part, b, my_shard_row[part]),
row_bytes[part],
ptr + b * batch_pitch,
token_pitch,
row_bytes[part],
s_local,
)
def place_replicated(stream, part, g):
# rows every rank holds go straight into my own slot
n = plan.replicated[part]
if not n:
return
ptr, batch_pitch, token_pitch = rep_src[part]
ptr += start(r, g, part) * head_bytes
row = plan.replicated_row(part)
for b in range(batch):
mem.copy2d(
stream,
slots.slot_ptr(pool, r, g, part, b, row),
row_bytes[part],
ptr + b * batch_pitch,
token_pitch,
row_bytes[part],
n,
)
ready = main.record_event()
own, sent = [], []
if fill is not None:
# group by group: the peers' blocks first, so their copies start
# while this rank fills its own block straight into its slot
block = lambda t: t.view(s_local, hg, head_dim)
own_rows = lambda part, g: slots.slot(pool, r, g, part)[
0, my_shard_row[part] : my_shard_row[part] + s_local
]
hg = plan.group_heads(0)
for g in range(groups):
for p in peers:
qk = bufs.send[p * groups + g]
fill(start(p, g), hg, qk[0], qk[1])
qk = slots.send_block(pool, p, g)
fill(start(p, g, 0), hg, qk[0], qk[1])
sent.append(main.record_event())
fill(start(r, g), hg, block(slot_of(r, g, 0)), block(slot_of(r, g, 1)))
fill(start(r, g, 0), hg, own_rows(0, g), own_rows(1, g))
own.append(main.record_event())
own_cin = []
cin.wait_event(ready)
with torch.cuda.stream(cin):
for g in range(groups):
for p in (r, *peers):
send_rows(cin, v, p, g, 2)
send_rows(cin, 2, p, g)
if fill is None:
send_rows(cin, q, p, g, 0)
send_rows(cin, k, p, g, 1)
send_rows(cin, 0, p, g)
send_rows(cin, 1, p, g)
if p == r:
for part in range(3):
place_replicated(cin, part, g)
own_cin.append(cin.record_event())
if fill is not None:
cin.wait_event(sent[g])
chunk = s_local * row_bytes[0]
for p in peers:
q_slot = slots.slot_ptr(pool, p, g, 0, 0, my_shard_row[0])
k_slot = slots.slot_ptr(pool, p, g, 1, 0, my_shard_row[1])
# q then k: two rows of `chunk`, one part apart in the slot
mem.copy2d(
cin,
slot_of(p, g, 0).data_ptr(),
slot_of(p, g, 1).data_ptr() - slot_of(p, g, 0).data_ptr(),
bufs.send[p * groups + g].data_ptr(),
q_slot,
k_slot - q_slot,
slots.send_block(pool, p, g).data_ptr(),
chunk,
chunk,
2,
)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
mem.write(cin, pool.fin_ptr[p] + 4 * r, base + g + 1)
# Head block b = p * groups + g of the output is group g of rank p's
# heads. Each group stream writes its blocks as soon as they exist, so
# the merge overlaps the groups still attending instead of trailing them.
merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device)
merged_blocks = merged.view(s_local, world * groups, hg, head_dim)
merged_pitch = heads * head_dim * esz
hq = plan.group_heads(0)
merged = torch.empty(
batch,
plan.out_rows,
plan.heads[0],
head_dim,
dtype=q.dtype,
device=q.device,
)
merged_ptr = merged.data_ptr()
merged_row = plan.heads[0] * head_bytes
out_row = row_bytes[0]
block_batch = plan.out_rows * out_row
shard_out, rep_out = plan.out_shard_row, plan.out_replicated_row
n_rep, rep_row = plan.replicated[0], plan.replicated_row(0)
done = []
for g in range(groups):
stream = self.pipe_group_streams[g]
@@ -452,37 +725,60 @@ class IpcA2AMultiState:
stream.wait_event(own_cin[g])
with torch.cuda.stream(stream):
for p in peers:
mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1)
part = lambda i: bufs.inb[r][slot, g, i].view(
world * s_local, hg, head_dim
)
out = attend(part(0), part(1), part(2)).contiguous()
rows = lambda p: out[p * s_local : (p + 1) * s_local]
mem.wait(stream, pool.fin_ptr[r] + 4 * p, base + g + 1)
views = [slots.slot(pool, r, g, part) for part in range(3)]
if squeeze:
out = attend(*(t[0] for t in views)).contiguous()[None]
else:
out = attend(*views).contiguous()
out_ptr = out.data_ptr()
out_batch = out.stride(0) * esz
for p in peers:
bufs.outb[p][slot, r, g].copy_(
rows(p).reshape(-1), non_blocking=True
)
# this peer's rows and the replicated rows every rank needs,
# one contiguous run per batch row
block = slots.out_block_ptr(pool, p, r, g)
runs = [(shard_out, plan.shard_row(0, p), s_local)]
if n_rep:
runs.append((rep_out, rep_row, n_rep))
for dst_row, src_row, n in runs:
mem.copy2d(
stream,
block + dst_row * out_row,
block_batch,
out_ptr + src_row * out_row,
out_batch,
n * out_row,
batch,
)
for p in peers:
mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call)
mem.copy2d(
stream,
merged_blocks[:, r * groups + g].data_ptr(),
merged_pitch,
rows(r).data_ptr(),
width,
width,
s_local,
)
mem.write(stream, pool.fout_ptr[p] + 4 * (g * world + r), call)
own_head = merged_ptr + start(r, g, 0) * head_bytes
for b in range(batch):
runs = [(shard_out, my_shard_row[0], s_local)]
if n_rep:
runs.append((rep_out, rep_row, n_rep))
for dst_row, src_row, n in runs:
mem.copy2d(
stream,
own_head + (b * plan.out_rows + dst_row) * merged_row,
merged_row,
out_ptr + b * out_batch + src_row * out_row,
out_row,
out_row,
n,
)
for p in peers:
mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call)
mem.wait(stream, pool.fout_ptr[r] + 4 * (g * world + p), call)
# the block and the merged output both have one row pitch
# across the batch, so a single copy covers every batch row
mem.copy2d(
stream,
merged_blocks[:, p * groups + g].data_ptr(),
merged_pitch,
bufs.outb[r][slot, p, g].data_ptr(),
width,
width,
s_local,
merged_ptr + start(p, g, 0) * head_bytes,
merged_row,
slots.out_block_ptr(pool, r, p, g),
out_row,
out_row,
batch * plan.out_rows,
)
done.append(stream.record_event())
for event in done:
@@ -490,15 +786,78 @@ class IpcA2AMultiState:
# the next call's fill rewrites `send` and the caller may free `v`, both
# read by the copy stream
main.wait_stream(cin)
if squeeze:
merged = merged[0]
if check is not None:
return self._first_sight(check, merged, reference)
return merged
def _fits_on_every_rank(self, nbytes: int) -> bool:
def _plan(self, q, k, v, replicated, replicated_first, groups, filled):
"""The plan these shapes pipeline with, or None; rank-independent."""
world = self.world
batch, s_local, _, head_dim = q.shape
parts = (q, k, v)
rows = tuple(0 if t is None else t.shape[1] for t in replicated)
if (
k.shape != v.shape
or any(
t.shape[:2] != (batch, s_local) or t.shape[3] != head_dim for t in parts
)
or rows[1] != rows[2]
or any(
t is not None and t.shape != (batch, n, parts[i].shape[2], head_dim)
for i, (t, n) in enumerate(zip(replicated, rows))
)
or (filled and (batch != 1 or any(rows) or q.shape != k.shape))
):
return None
plan = _PipelinePlan(
batch=batch,
s_local=s_local,
heads=tuple(t.shape[2] for t in parts),
head_dim=head_dim,
replicated=rows,
replicated_first=replicated_first,
world=world,
groups=groups,
)
if groups < 0:
plan = dataclasses.replace(plan, groups=_auto_pipeline_groups(plan))
if not plan.groups or any(h % (world * plan.groups) for h in plan.heads):
return None
return plan
def _first_sight(self, check, merged, reference):
"""Keep the pipeline for `check` only if it matches the sequential
exchange's `reference` byte for byte on every rank; a group collective."""
same = torch.tensor(
[int(torch.equal(merged, reference))], dtype=torch.int32, device="cuda"
)
dist.all_reduce(same, op=dist.ReduceOp.MIN, group=self.group)
if same.item():
self.verified.add(check)
return merged
self.mismatched.add(check)
plan = check[0][1]
logger.info(
"Ulysses pipeline over %d head groups differs from the sequential "
"exchange for %s under %s; keeping the sequential exchange",
plan.groups,
plan,
check[1],
)
return reference
def _fits_on_every_rank(self, nbytes: int, reclaimable: int = 0) -> bool:
"""Whether every rank can spare twice `nbytes`, counting the allocator's
cached blocks; a group collective, so all ranks reach the same answer."""
cached blocks and `reclaimable` bytes it is about to free; a group
collective, so all ranks reach the same answer."""
free, _ = torch.cuda.mem_get_info()
cached = torch.cuda.memory_reserved() - torch.cuda.memory_allocated()
fits = torch.tensor(
[int(free + cached >= 2 * nbytes)], dtype=torch.int32, device="cuda"
[int(free + cached + reclaimable >= 2 * nbytes)],
dtype=torch.int32,
device="cuda",
)
dist.all_reduce(fits, op=dist.ReduceOp.MIN, group=self.group)
return bool(fits.item())
@@ -598,12 +957,35 @@ def ipc_a2a_multi_ready(group, enabled: bool | None = None) -> bool:
return True
def ulysses_pipelined_attention(q, k, v, attend, groups: int, fill=None):
def ulysses_pipelined_attention(
q,
k,
v,
attend,
groups: int,
fill=None,
sequential=None,
signature=None,
replicated=(None, None, None),
replicated_first: bool = True,
):
"""Pipelined Ulysses attention over the copy-engine transport, or None
when the Ulysses group cannot use it (the caller keeps its NCCL path)."""
when the Ulysses group cannot use it (the caller keeps its NCCL path).
See ``IpcA2AMultiState.pipelined_attention`` for the arguments."""
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
group = get_sp_group().ulysses_group
if group is None or not ipc_a2a_multi_ready(group, enabled=True):
return None
return IPC_A2A_MULTI.pipelined_attention(q, k, v, attend, groups, fill=fill)
return IPC_A2A_MULTI.pipelined_attention(
q,
k,
v,
attend,
groups,
fill=fill,
sequential=sequential,
signature=signature,
replicated=replicated,
replicated_first=replicated_first,
)
@@ -68,6 +68,7 @@ from sglang.multimodal_gen.runtime.layers.usp import (
_usp_output_all_to_all,
_usp_output_all_to_all_varlen,
ring_attn,
uncached_a2a_staging,
)
from sglang.multimodal_gen.runtime.managers.forward_context import (
ForwardContext,
@@ -95,6 +96,14 @@ _VARLEN_FA_ENABLED = os.environ.get("SGLANG_VARLEN_FA", "1") != "0"
# and run dense attention on the padded layout.
_SP_PAD_MASK_DISABLED = envs.SGLANG_DIFFUSION_DISABLE_SP_PAD_MASK
# Dense backends the head-group pipeline runs against: each computes a head
# independently, so a head group comes out as those heads do in the full call
# (FlashAttention on Hopper, cuDNN SDPA on Blackwell).
PIPELINED_ATTENTION_BACKENDS = (
AttentionBackendEnum.FA,
AttentionBackendEnum.DYNAMIC_CUDNN_SDPA,
)
def _resolve_sp_attention_mode(
*, causal: bool, sparse_backend: bool
@@ -890,6 +899,7 @@ class USPAttention(nn.Module):
self.skip_sequence_parallel = skip_sequence_parallel
self.enable_packed_qkv_input_a2a = bool(enable_packed_qkv_input_a2a)
self._pipeline_declined: set = set()
self.sp_attention_mode, self.sp_attention_mode_is_auto = (
_resolve_sp_attention_mode(
causal=causal, sparse_backend=self.backend.is_sparse
@@ -1259,14 +1269,10 @@ class USPAttention(nn.Module):
)
sp_size = get_ulysses_parallel_world_size()
if sp_size > 1 and not qkv_pre_all_to_all:
qkv_fast = _ipc_input_a2a_qkv(q, k, v)
if qkv_fast is not None:
q, k, v = qkv_fast
else:
q, k, v = _usp_input_all_to_all_qkv(q, k, v)
if (
exchange = sp_size > 1 and not qkv_pre_all_to_all
# the gathered sequence the attention below runs over
gathered_seq = q.shape[1] * (sp_size if exchange else 1)
tail_fa = (
_VARLEN_FA_ENABLED
and self.backend == AttentionBackendEnum.FA
and meta_pad_start is not None
@@ -1274,109 +1280,121 @@ class USPAttention(nn.Module):
and meta_pad_end > meta_pad_start
and q.device.type == "cuda"
and q.dtype in (torch.float16, torch.bfloat16)
):
bs, seq = q.shape[0], q.shape[1]
assert 0 <= meta_pad_start < meta_pad_end <= seq
cu_tail = attn_mask_meta.get("cu_seqlens_tail")
if cu_tail is not None and meta_pad_end == seq:
# Zero-copy tail path: run varlen FA straight over the
# padded layout, each row split into [valid | pad] segments
# (contiguous reshapes only, no repacking).
assert cu_tail.numel() == 2 * bs + 1, (
"cu_seqlens_tail does not match the batch size"
)
gathered_mask = gathered_mask_meta = None
if not tail_fa:
# If NCCL timeout/deadlock occurs here, check whether
# attn_mask is inconsistent across SP ranks (None on some,
# Tensor on others), which causes all_gather participant
# mismatch. Upstream mask builders must ensure all ranks
# produce the same mask type.
if attn_mask is None:
# Meta-only tail-pad caller on a non-FA fallback: the
# gathered mask is fully determined by the pad span, no
# collective needed.
gathered_mask = torch.ones(
q.shape[0], gathered_seq, dtype=torch.bool, device=q.device
)
out = flash_attn_varlen_func(
q=q.reshape(bs * seq, *q.shape[2:]),
k=k.reshape(bs * seq, *k.shape[2:]),
v=v.reshape(bs * seq, *v.shape[2:]),
cu_seqlens_q=cu_tail,
cu_seqlens_k=cu_tail,
max_seqlen_q=attn_mask_meta["max_seqlen_tail"],
max_seqlen_k=attn_mask_meta["max_seqlen_tail"],
gathered_mask[:, meta_pad_start:meta_pad_end] = False
else:
gathered_mask = sequence_model_parallel_all_gather(
attn_mask.contiguous(), dim=1
)
gathered_rows = (q.shape[0], gathered_seq)
if (
_VARLEN_FA_ENABLED
and self.backend == AttentionBackendEnum.FA
and gathered_mask.dtype
in (torch.bool, torch.uint8, torch.int32, torch.int64)
and q.device.type == "cuda"
and gathered_mask.device == q.device
and q.dtype in (torch.float16, torch.bfloat16)
and gathered_rows == gathered_mask.shape
and k.shape[:2] == v.shape[:2] == q.shape[:2]
):
gathered_mask_meta = build_varlen_mask_meta(gathered_mask)
assert gathered_mask_meta["inv_indices"].shape[0] == (
gathered_rows[0] * gathered_rows[1]
), "gathered attn_mask shape does not match q/k/v"
if gathered_mask_meta["indices"].shape[0] == 0:
gathered_mask_meta = None
def masked_attention(q, k, v):
"""Attention over the gathered sequence; heads are independent."""
bs, seq = q.shape[0], q.shape[1]
if tail_fa:
assert 0 <= meta_pad_start < meta_pad_end <= seq
cu_tail = attn_mask_meta.get("cu_seqlens_tail")
if cu_tail is not None and meta_pad_end == seq:
# Zero-copy tail path: run varlen FA straight over the
# padded layout, each row split into [valid | pad]
# segments (contiguous reshapes only, no repacking).
assert cu_tail.numel() == 2 * bs + 1, (
"cu_seqlens_tail does not match the batch size"
)
out = flash_attn_varlen_func(
q=q.reshape(bs * seq, *q.shape[2:]),
k=k.reshape(bs * seq, *k.shape[2:]),
v=v.reshape(bs * seq, *v.shape[2:]),
cu_seqlens_q=cu_tail,
cu_seqlens_k=cu_tail,
max_seqlen_q=attn_mask_meta["max_seqlen_tail"],
max_seqlen_k=attn_mask_meta["max_seqlen_tail"],
softmax_scale=self.softmax_scale,
causal=False,
ver=_fa_backend.fa_ver,
).reshape(bs, seq, *q.shape[2:])
# Match the packed paths: masked query rows read as zeros.
out[:, meta_pad_start:].zero_()
return out
valid_seq = seq - (meta_pad_end - meta_pad_start)
q_dense = torch.cat(
[q[:, :meta_pad_start], q[:, meta_pad_end:]], dim=1
)
k_dense = torch.cat(
[k[:, :meta_pad_start], k[:, meta_pad_end:]], dim=1
)
v_dense = torch.cat(
[v[:, :meta_pad_start], v[:, meta_pad_end:]], dim=1
)
cu_seqlens = torch.arange(
0,
(bs + 1) * valid_seq,
valid_seq,
dtype=torch.int32,
device=q.device,
)
out_dense = flash_attn_varlen_func(
q=q_dense.reshape(bs * valid_seq, *q.shape[2:]),
k=k_dense.reshape(bs * valid_seq, *k.shape[2:]),
v=v_dense.reshape(bs * valid_seq, *v.shape[2:]),
cu_seqlens_q=cu_seqlens,
cu_seqlens_k=cu_seqlens,
max_seqlen_q=valid_seq,
max_seqlen_k=valid_seq,
softmax_scale=self.softmax_scale,
causal=False,
ver=_fa_backend.fa_ver,
).reshape(bs, seq, *q.shape[2:])
# Match the packed paths: masked query rows read as zeros.
out[:, meta_pad_start:].zero_()
if sp_size > 1:
out = _usp_output_all_to_all(out, head_dim=2)
return out
valid_seq = seq - (meta_pad_end - meta_pad_start)
q_dense = torch.cat([q[:, :meta_pad_start], q[:, meta_pad_end:]], dim=1)
k_dense = torch.cat([k[:, :meta_pad_start], k[:, meta_pad_end:]], dim=1)
v_dense = torch.cat([v[:, :meta_pad_start], v[:, meta_pad_end:]], dim=1)
cu_seqlens = torch.arange(
0,
(bs + 1) * valid_seq,
valid_seq,
dtype=torch.int32,
device=q.device,
)
out_dense = flash_attn_varlen_func(
q=q_dense.reshape(bs * valid_seq, *q.shape[2:]),
k=k_dense.reshape(bs * valid_seq, *k.shape[2:]),
v=v_dense.reshape(bs * valid_seq, *v.shape[2:]),
cu_seqlens_q=cu_seqlens,
cu_seqlens_k=cu_seqlens,
max_seqlen_q=valid_seq,
max_seqlen_k=valid_seq,
softmax_scale=self.softmax_scale,
causal=False,
ver=_fa_backend.fa_ver,
).reshape(bs, valid_seq, *q.shape[2:])
gap_out = out_dense.new_zeros(
bs,
meta_pad_end - meta_pad_start,
out_dense.shape[2],
out_dense.shape[3],
)
out = torch.cat(
[
out_dense[:, :meta_pad_start],
gap_out,
out_dense[:, meta_pad_start:],
],
dim=1,
)
if sp_size > 1:
out = _usp_output_all_to_all(out, head_dim=2)
return out
).reshape(bs, valid_seq, *q.shape[2:])
gap_out = out_dense.new_zeros(
bs,
meta_pad_end - meta_pad_start,
out_dense.shape[2],
out_dense.shape[3],
)
return torch.cat(
[
out_dense[:, :meta_pad_start],
gap_out,
out_dense[:, meta_pad_start:],
],
dim=1,
)
# If NCCL timeout/deadlock occurs here, check whether
# attn_mask is inconsistent across SP ranks (None on some, Tensor on
# others), which causes all_gather participant mismatch. Upstream
# mask builders must ensure all ranks produce the same mask type.
if attn_mask is None:
# Meta-only tail-pad caller on a non-FA fallback: the gathered
# mask is fully determined by the pad span, no collective needed.
gathered_mask = torch.ones(
q.shape[0], q.shape[1], dtype=torch.bool, device=q.device
)
gathered_mask[:, meta_pad_start:meta_pad_end] = False
else:
gathered_mask = sequence_model_parallel_all_gather(
attn_mask.contiguous(), dim=1
)
if (
_VARLEN_FA_ENABLED
and self.backend == AttentionBackendEnum.FA
and gathered_mask.dtype
in (torch.bool, torch.uint8, torch.int32, torch.int64)
and q.device.type == "cuda"
and gathered_mask.device == q.device
and q.dtype in (torch.float16, torch.bfloat16)
and q.shape[:2] == gathered_mask.shape == k.shape[:2] == v.shape[:2]
):
bs, seq = q.shape[0], q.shape[1]
gathered_mask_meta = build_varlen_mask_meta(gathered_mask)
indices = gathered_mask_meta["indices"]
inv_indices = gathered_mask_meta["inv_indices"]
assert inv_indices.shape[0] == bs * seq, (
"gathered attn_mask shape does not match q/k/v"
)
if indices.shape[0] > 0:
q_unpad, k_unpad, v_unpad = fused_pack_qkv(q, k, v, indices)
if gathered_mask_meta is not None:
q_unpad, k_unpad, v_unpad = fused_pack_qkv(
q, k, v, gathered_mask_meta["indices"]
)
out_unpad = flash_attn_varlen_func(
q=q_unpad,
k=k_unpad,
@@ -1389,33 +1407,57 @@ class USPAttention(nn.Module):
causal=False,
ver=_fa_backend.fa_ver,
)
out = fused_scatter_to_padded(out_unpad, inv_indices, bs, seq)
if sp_size > 1:
out = _usp_output_all_to_all(out, head_dim=2)
return out
return fused_scatter_to_padded(
out_unpad, gathered_mask_meta["inv_indices"], bs, seq
)
q_ = q.transpose(1, 2)
k_ = k.transpose(1, 2)
v_ = v.transpose(1, 2)
mask = _prepare_sdpa_mask(gathered_mask, dtype=q_.dtype, device=q_.device)
sdpa_context = (
sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
if self.allow_cudnn_sdp and q_.device.type == "cuda"
else nullcontext()
)
with sdpa_context:
out = torch.nn.functional.scaled_dot_product_attention(
q_,
k_,
v_,
attn_mask=mask,
dropout_p=0.0,
is_causal=False,
scale=self.softmax_scale,
).transpose(1, 2)
if sp_size > 1:
out = _usp_output_all_to_all(out, head_dim=2)
return out
q_ = q.transpose(1, 2)
k_ = k.transpose(1, 2)
v_ = v.transpose(1, 2)
mask = _prepare_sdpa_mask(
gathered_mask, dtype=q_.dtype, device=q_.device
)
sdpa_context = (
sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
if self.allow_cudnn_sdp and q_.device.type == "cuda"
else nullcontext()
)
with sdpa_context:
return torch.nn.functional.scaled_dot_product_attention(
q_,
k_,
v_,
attn_mask=mask,
dropout_p=0.0,
is_causal=False,
scale=self.softmax_scale,
).transpose(1, 2)
def sequential():
q_all, k_all, v_all = q, k, v
if exchange:
qkv_fast = _ipc_input_a2a_qkv(q, k, v)
if qkv_fast is not None:
q_all, k_all, v_all = qkv_fast
else:
q_all, k_all, v_all = _usp_input_all_to_all_qkv(q, k, v)
out = masked_attention(q_all, k_all, v_all)
if sp_size > 1:
out = _usp_output_all_to_all(out, head_dim=2)
return out
if exchange:
out = self._pipelined_ulysses(
q,
k,
v,
masked_attention,
sequential,
layout="masked-tail" if tail_fa else "masked",
)
if out is not None:
return out
return sequential()
if effective_skip_sp or get_sequence_parallel_world_size() == 1:
# No sequence parallelism, just run local attention.
@@ -1444,6 +1486,27 @@ class USPAttention(nn.Module):
q, k, v, ctx_attn_metadata, num_replicated_kv_prefix
)
if (
sp_size > 1
and not qkv_pre_all_to_all
and get_ring_parallel_world_size() == 1
and not self.enable_packed_qkv_input_a2a
):
def attend(q_group, k_group, v_group):
return self.attn_impl.forward(
q_group, k_group, v_group, ctx_attn_metadata
)
def sequential():
qs, ks, vs = _usp_input_all_to_all_qkv(q, k, v)
out = self.attn_impl.forward(qs, ks, vs, ctx_attn_metadata)
return _usp_output_all_to_all(out, head_dim=2)
out = self._pipelined_ulysses(q, k, v, attend, sequential, layout="dense")
if out is not None:
return out
# Ulysses-style All-to-All for sequence/head sharding
if sp_size > 1 and not qkv_pre_all_to_all:
# -> [B, S, H_local, D]
@@ -1482,6 +1545,91 @@ class USPAttention(nn.Module):
return out
def _pipelined_ulysses(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
attend,
sequential,
*,
layout: str,
replicated=(None, None, None),
replicated_first: bool = True,
) -> torch.Tensor | None:
"""Ulysses attention with the exchange pipelined over head groups.
The building block every Ulysses path of this layer goes through: q/k/v
are the call's ``[B, S_local, H, D]`` sequence shards and ``replicated``
the rows every rank holds (see ``ulysses_pipelined_attention``).
``attend(q, k, v)`` is the path's attention over the gathered sequence,
given one head group at a time, and ``sequential()`` the path's own
exchange-attend-exchange: the first call of each shape runs both and
keeps the pipeline only if every rank sees the same bytes. Returns
None, so the caller runs ``sequential()`` itself, when this call
cannot pipeline.
"""
groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS
if (
not q.is_cuda
or groups in (0, 1)
or self.backend not in PIPELINED_ATTENTION_BACKENDS
or torch.compiler.is_compiling()
or torch.cuda.is_current_stream_capturing()
):
return None
# shapes the transport already turned down stay down (too short to
# split, buffers that do not fit, a backend that failed the first-sight
# check); skip straight to the sequential path for them
key = (
layout,
groups,
self.backend,
q.shape,
k.shape,
v.shape,
q.stride(),
k.stride(),
v.stride(),
tuple(None if t is None else (t.shape, t.stride()) for t in replicated),
replicated_first,
)
if key in self._pipeline_declined:
return None
if (
get_ring_parallel_world_size() > 1
or get_request_skip_softmax_params() is not None
):
return None
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
ulysses_pipelined_attention,
)
def reference():
# once per shape: leave no staging behind that later calls never use
with uncached_a2a_staging():
return sequential()
out = ulysses_pipelined_attention(
q,
k,
v,
attend,
groups,
sequential=reference,
signature=(
type(self.attn_impl).__name__,
self.causal,
self.softmax_scale,
layout,
),
replicated=replicated,
replicated_first=replicated_first,
)
if out is None:
self._pipeline_declined.add(key)
return out
def _forward_ring_tail_pad(
self,
q: torch.Tensor,
@@ -1740,8 +1888,6 @@ class USPAttention(nn.Module):
k_rep, k_shard = k[:, :num_rep], k[:, num_rep:]
v_rep, v_shard = v[:, :num_rep], v[:, num_rep:]
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
joint_mask = joint_mask_meta = None
if attn_mask is not None:
cache_key = (
@@ -1769,44 +1915,68 @@ class USPAttention(nn.Module):
joint_mask_meta,
)
# Q and KV can have different head counts (GQA), so slice each replicated
# prefix by its own per-rank head shard to match the all-to-all'd suffix.
# For MHA (kv heads == q heads) this is identical to the q shard.
h_local = q_shard.shape[2]
kv_h_local = k_shard.shape[2]
h_start = u_rank * h_local
kv_h_start = u_rank * kv_h_local
q_rep = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
k_rep = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
v_rep = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
def attend(q_group, k_group, v_group):
if joint_mask is not None:
return self._masked_local_attention(
q_group,
k_group,
v_group,
joint_mask,
attn_mask_meta=joint_mask_meta,
)
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
q = torch.cat([q_rep, q_shard], dim=1)
out = self._replicated_kv_attention(
q,
def sequential():
q_gathered, k_gathered, v_gathered = _usp_input_all_to_all_qkv(
q_shard, k_shard, v_shard
)
# Q and KV can have different head counts (GQA), so slice each
# replicated prefix by its own per-rank head shard to match the
# all-to-all'd suffix. For MHA (kv heads == q heads) this is
# identical to the q shard.
h_local = q_gathered.shape[2]
kv_h_local = k_gathered.shape[2]
h_start = u_rank * h_local
kv_h_start = u_rank * kv_h_local
q_local = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
k_local = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
v_local = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
out = self._replicated_kv_attention(
torch.cat([q_local, q_gathered], dim=1),
k_gathered,
v_gathered,
k_local,
v_local,
ctx_attn_metadata,
attn_mask=joint_mask,
attn_mask_meta=joint_mask_meta,
)
out_rep = out[:, :num_rep]
out_shard = _usp_output_all_to_all(out[:, num_rep:], head_dim=2)
if sp_size > 1:
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
torch.distributed.all_gather(
gathered,
out_rep.contiguous(),
group=get_sp_group().ulysses_group,
)
out_rep = torch.cat(gathered, dim=2)
return torch.cat([out_rep, out_shard], dim=1)
out = self._pipelined_ulysses(
q_shard,
k_shard,
v_shard,
k_rep,
v_rep,
ctx_attn_metadata,
attn_mask=joint_mask,
attn_mask_meta=joint_mask_meta,
attend,
sequential,
layout="replicated-prefix" + ("-masked" if joint_mask is not None else ""),
replicated=(q_rep, k_rep, v_rep),
)
out_rep = out[:, :num_rep]
out_shard = out[:, num_rep:]
out_shard = _usp_output_all_to_all(out_shard, head_dim=2)
if sp_size > 1:
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
torch.distributed.all_gather(
gathered,
out_rep.contiguous(),
group=get_sp_group().ulysses_group,
)
out_rep = torch.cat(gathered, dim=2)
return torch.cat([out_rep, out_shard], dim=1)
return out if out is not None else sequential()
def _replicated_kv_attention(
self,
@@ -1998,32 +2168,52 @@ class USPAttention(nn.Module):
"""split form avoids materializing full K/V before Ulysses all-to-all"""
u_rank = get_ulysses_parallel_rank()
if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1:
q, k_shard, v_shard = async_a2a_communicate(
[q, k_shard, v_shard],
get_ulysses_parallel_world_size(),
get_sp_group().ulysses_group,
self._get_usp_a2a_stream(),
local_seq_2_local_head=True,
def attend(q_group, k_group, v_group):
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
def sequential():
if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1:
q_gathered, k_gathered, v_gathered = async_a2a_communicate(
[q, k_shard, v_shard],
get_ulysses_parallel_world_size(),
get_sp_group().ulysses_group,
self._get_usp_a2a_stream(),
local_seq_2_local_head=True,
)
q_gathered = q_gathered.contiguous()
k_gathered = k_gathered.contiguous()
v_gathered = v_gathered.contiguous()
else:
q_gathered = _usp_input_all_to_all(q, head_dim=2)
k_gathered = _usp_input_all_to_all(k_shard, head_dim=2)
v_gathered = _usp_input_all_to_all(v_shard, head_dim=2)
h_kv_local = k_gathered.shape[2]
h_start = u_rank * h_kv_local
h_end = h_start + h_kv_local
k_local = k_rep[:, :, h_start:h_end, :].contiguous()
v_local = v_rep[:, :, h_start:h_end, :].contiguous()
out = self._replicated_kv_attention(
q_gathered,
k_gathered,
v_gathered,
k_local,
v_local,
ctx_attn_metadata,
)
q = q.contiguous()
k_shard = k_shard.contiguous()
v_shard = v_shard.contiguous()
else:
q = _usp_input_all_to_all(q, head_dim=2)
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
return _usp_output_all_to_all(out, head_dim=2)
h_kv_local = k_shard.shape[2]
h_start = u_rank * h_kv_local
h_end = h_start + h_kv_local
k_rep = k_rep[:, :, h_start:h_end, :].contiguous()
v_rep = v_rep[:, :, h_start:h_end, :].contiguous()
out = self._replicated_kv_attention(
q, k_shard, v_shard, k_rep, v_rep, ctx_attn_metadata
out = self._pipelined_ulysses(
q,
k_shard,
v_shard,
attend,
sequential,
layout="replicated-kv-prefix",
replicated=(None, k_rep, v_rep),
)
return _usp_output_all_to_all(out, head_dim=2)
return out if out is not None else sequential()
def _forward_with_replicated_suffix(
self,
@@ -2049,37 +2239,58 @@ class USPAttention(nn.Module):
k_shard, k_rep = k[:, :-num_rep], k[:, -num_rep:]
v_shard, v_rep = v[:, :-num_rep], v[:, -num_rep:]
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
def attend(q_group, k_group, v_group):
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
h_local = q_shard.shape[2]
kv_h_local = k_shard.shape[2]
h_start = u_rank * h_local
kv_h_start = u_rank * kv_h_local
q_rep = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
k_rep = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
v_rep = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
q = torch.cat([q_shard, q_rep], dim=1)
out = self._replicated_kv_attention(
q, k_shard, v_shard, k_rep, v_rep, ctx_attn_metadata, rep_first=False
)
out_shard = out[:, :-num_rep]
out_rep = out[:, -num_rep:]
out_shard = _usp_output_all_to_all(out_shard, head_dim=2)
sp_size = get_ulysses_parallel_world_size()
if sp_size > 1:
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
torch.distributed.all_gather(
gathered,
out_rep.contiguous(),
group=get_sp_group().ulysses_group,
def sequential():
q_gathered, k_gathered, v_gathered = _usp_input_all_to_all_qkv(
q_shard, k_shard, v_shard
)
out_rep = torch.cat(gathered, dim=2)
return torch.cat([out_shard, out_rep], dim=1)
h_local = q_gathered.shape[2]
kv_h_local = k_gathered.shape[2]
h_start = u_rank * h_local
kv_h_start = u_rank * kv_h_local
q_local = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
k_local = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
v_local = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
out = self._replicated_kv_attention(
torch.cat([q_gathered, q_local], dim=1),
k_gathered,
v_gathered,
k_local,
v_local,
ctx_attn_metadata,
rep_first=False,
)
out_shard = _usp_output_all_to_all(out[:, :-num_rep], head_dim=2)
out_rep = out[:, -num_rep:]
sp_size = get_ulysses_parallel_world_size()
if sp_size > 1:
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
torch.distributed.all_gather(
gathered,
out_rep.contiguous(),
group=get_sp_group().ulysses_group,
)
out_rep = torch.cat(gathered, dim=2)
return torch.cat([out_shard, out_rep], dim=1)
out = self._pipelined_ulysses(
q_shard,
k_shard,
v_shard,
attend,
sequential,
layout="replicated-suffix",
replicated=(q_rep, k_rep, v_rep),
replicated_first=False,
)
return out if out is not None else sequential()
class _BCGBoxedTupleOutput:
@@ -1,5 +1,6 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
import contextlib
import logging
import math
from typing import TYPE_CHECKING
@@ -28,6 +29,25 @@ logger = logging.getLogger(__name__)
_A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {}
# depth of open uncached_a2a_staging() blocks
_UNCACHED_STAGING = 0
@contextlib.contextmanager
def uncached_a2a_staging():
"""Run the Ulysses exchanges inside the block over NCCL on per-call buffers.
For a one-off exchange the steady state never repeats (the pipelined
attention's first-sight reference): the staging cache and the IPC
transports' staging pairs would otherwise stay allocated at its size. An
all-to-all is a permutation, so the transport never changes its bytes.
"""
global _UNCACHED_STAGING
_UNCACHED_STAGING += 1
try:
yield
finally:
_UNCACHED_STAGING -= 1
def drop_a2a_staging_buffers() -> None:
@@ -59,7 +79,8 @@ def _a2a_staging_buffer(
not be shared with eager replays.
"""
if (
torch.is_grad_enabled()
_UNCACHED_STAGING
or torch.is_grad_enabled()
or torch.compiler.is_compiling()
or device.type != "cuda"
or torch.cuda.is_current_stream_capturing()
@@ -88,7 +109,7 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
assert ulysses_pg is not None, "Ulysses process group is not initialized."
x_shape = x.shape
x = x.flatten().contiguous()
if x.is_cuda:
if x.is_cuda and not _UNCACHED_STAGING:
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
IPC_A2A_MULTI,
ipc_a2a_multi_ready,
@@ -148,6 +169,8 @@ def _ipc_ready_group():
ipc_a2a_ready,
)
if _UNCACHED_STAGING:
return None
group = get_sp_group().ulysses_group
return group if ipc_a2a_ready(group) else None
@@ -1562,6 +1562,7 @@ STANDALONE_FILES = {
"../single_test_file/test_ar_models.py",
"../single_test_file/test_ipc_a2a_2_gpu.py",
"../single_test_file/test_ipc_a2a_multi_gpu.py",
"../single_test_file/test_usp_pipelined_attention_multi_gpu.py",
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py",
"../single_test_file/test_encoder_fold_srt_2_gpu.py",
"../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py",
@@ -1605,6 +1606,8 @@ STANDALONE_FILE_EST_TIMES = {
"../single_test_file/test_ipc_a2a_2_gpu.py": 240.0,
# 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT
"../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0,
# 2 ranks here (4 ranks skip below four GPUs); ~45 s on B300 with warm JIT
"../single_test_file/test_usp_pipelined_attention_multi_gpu.py": 120.0,
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0,
"../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0,
# ~60 s locally with a warm HF cache (load + one capture + 4 steps);
@@ -27,6 +27,24 @@ def _attend(q, k, v):
return (q.float() * 2 + k.flip(0).float() - v.roll(1, 0).float()).to(q.dtype)
def _attend_rows(q, k, v):
"""`_attend` for [batch, rows, heads, head_dim] parts whose k/v may carry more
rows (a replicated KV prefix) and fewer heads (GQA) than q."""
rep = q.shape[2] // k.shape[2]
k = k.repeat_interleave(rep, dim=2).float()
v = v.repeat_interleave(rep, dim=2).float()
rows = q.shape[1]
# elementwise only (a reduction could round differently per head count):
# the leading and the trailing rows both read, so every row counts
return (
q.float() * 2
+ k[:, :rows]
+ k.flip(1)[:, :rows]
- v.roll(1, 1)[:, -rows:]
+ v[:, :rows] * 0.5
).to(q.dtype)
def _worker() -> int:
import torch.distributed as dist
@@ -69,7 +87,7 @@ def _worker() -> int:
if world == 2:
cases.append((128, 16, 64, 8))
for s_local, heads, head_dim, groups in cases:
# three calls in a row: the slots alternate and the counters advance
# three calls in a row: the slots are reused and the counters advance
for call in range(3):
torch.manual_seed(
1000 * call + s_local
@@ -132,23 +150,146 @@ def _worker() -> int:
f"filled pipeline {(s_local, heads, head_dim, groups)} call {call}"
)
# groups=-1 picks a count that divides the heads per rank (4 for 16 heads at
# 2 ranks, 2 at 4) and still matches
# groups=-1 picks the most groups that divide the heads per rank and whose
# calls still fill four waves of 128-row tiles, and still matches; a shape too
# small to fill them keeps the sequential exchange
sms = torch.cuda.get_device_properties(rank).multi_processor_count
s_auto = 4 * sms * 128 // world # one head per group fills four waves
torch.manual_seed(77)
full = [
torch.randn(64 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
torch.randn(s_auto * world, 16, 64, dtype=torch.bfloat16, device="cuda")
for _ in range(3)
]
q, k, v = (t.narrow(0, rank * 64, 64).contiguous() for t in full)
q, k, v = (t.narrow(0, rank * s_auto, s_auto).contiguous() for t in full)
got = ulysses_pipelined_attention(q, k, v, _attend, -1)
if got is None or not torch.equal(sequential(q, k, v), got):
failures.append("auto group count")
small = torch.randn(64, 16, 64, dtype=torch.bfloat16, device="cuda")
if ulysses_pipelined_attention(small, small, small, _attend, -1) is not None:
failures.append("auto pipelined a shape too small to fill the GPU")
# sequential=: the first call per shape and signature also runs the
# sequential exchange and keeps the pipeline only on a byte-for-byte match
torch.manual_seed(78)
full = [
torch.randn(80 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
for _ in range(3)
]
q, k, v = (t.narrow(0, rank * 80, 80).contiguous() for t in full)
ref = sequential(q, k, v)
for call in range(2):
got = ulysses_pipelined_attention(
q,
k,
v,
_attend,
2,
sequential=lambda: sequential(q, k, v),
signature="same",
)
if got is None or not torch.equal(ref, got):
failures.append(f"verified pipeline call {call}")
off = ref + 1 # a "sequential" result the pipeline cannot reproduce
got = ulysses_pipelined_attention(
q, k, v, _attend, 2, sequential=lambda: off, signature="other"
)
if got is None or not torch.equal(got, off):
failures.append("a mismatching first call did not return the sequential result")
if (
ulysses_pipelined_attention(
q, k, v, _attend, 2, sequential=lambda: off, signature="other"
)
is not None
):
failures.append("a mismatched shape kept pipelining")
# a head count the groups cannot split must decline, not mis-shard
q = torch.randn(32, 12, 64, dtype=torch.bfloat16, device="cuda")
if ulysses_pipelined_attention(q, q, q, _attend, 4) is not None:
failures.append("pipelined accepted 12 heads in 4 groups")
# batch, replicated rows (prefix, suffix, KV-only prefix) and GQA: the output
# rows match attention over the global layout, computed with every head
def layout_case(batch, s_local, heads, kv_heads, rep, rep_kv, first, groups, seed):
torch.manual_seed(seed)
dims = 32
shard = lambda h: torch.randn(
batch, s_local * world, h, dims, device="cuda"
).bfloat16()
full = [shard(heads), shard(kv_heads), shard(kv_heads)]
reps = [
torch.randn(batch, n, h, dims, device="cuda").bfloat16() if n else None
for n, h in ((rep, heads), (rep_kv, kv_heads), (rep_kv, kv_heads))
]
mine = [t[:, rank * s_local : (rank + 1) * s_local].contiguous() for t in full]
def joined(t, r):
if r is None:
return t
return torch.cat([r, t] if first else [t, r], dim=1)
out_all = _attend_rows(*(joined(t, r) for t, r in zip(full, reps)))
lo = (rep if first else 0) + rank * s_local
own = out_all[:, lo : lo + s_local]
if rep:
rep_rows = out_all[:, :rep] if first else out_all[:, -rep:]
own = torch.cat([rep_rows, own] if first else [own, rep_rows], dim=1)
got = ulysses_pipelined_attention(
*mine,
_attend_rows,
groups,
replicated=tuple(reps),
replicated_first=first,
)
label = f"layout b{batch} h{heads}/{kv_heads} rep {rep}/{rep_kv} first={first} g{groups}"
if got is None:
failures.append(f"{label} returned None")
elif not torch.equal(got, own):
failures.append(label)
for call in range(2): # every layout reuses the one pool
layout_case(2, 48, 16, 16, 0, 0, True, 2, 300 + call)
layout_case(1, 40, 16, 16, 9, 9, True, 2, 310 + call)
layout_case(2, 40, 16, 16, 7, 7, False, 2, 320 + call)
layout_case(1, 40, 16, 16, 0, 11, True, 2, 330 + call)
layout_case(2, 40, 16, 8, 5, 5, True, 2, 340 + call)
# one pool per dtype serves every plan above, sized for the largest of them
pools = IPC_A2A_MULTI.pools
if list(pools) != [torch.bfloat16]:
failures.append(f"pipeline pools per dtype: {list(pools)}")
else:
slots = list(IPC_A2A_MULTI.slots.values())
widest = (
max(s.in_total for s in slots),
max(s.out_total for s in slots),
max(s.plan.groups for s in slots),
)
if pools[torch.bfloat16].sizes != widest:
failures.append(f"pool sizes {pools[torch.bfloat16].sizes} != {widest}")
# one set of slots is enough: a peer cannot write a call's rows or output
# blocks before this rank has read the previous call's, however far ahead
# it runs. Rank 0 attends slowly; alternating plans share the pool.
def slow_attend(q, k, v):
if rank == 0:
torch.cuda._sleep(20_000_000)
return _attend(q, k, v)
issued = []
for call in range(6): # queued back to back; compared only afterwards
s_local, heads, groups = ((96, 16, 2), (40, 32, 4))[call % 2]
torch.manual_seed(4000 + call)
full = [
torch.randn(s_local * world, heads, 64, dtype=torch.bfloat16, device="cuda")
for _ in range(3)
]
parts = [t.narrow(0, rank * s_local, s_local).contiguous() for t in full]
issued.append((parts, ulysses_pipelined_attention(*parts, slow_attend, groups)))
for call, (parts, got) in enumerate(issued):
if got is None or not torch.equal(sequential(*parts), got):
failures.append(f"skewed back-to-back call {call}")
# the plain N-rank exchange behind _usp_all_to_all_single
for numel in (world * 1024, world * 4096 * 33):
x = torch.randn(numel, dtype=torch.bfloat16, device="cuda")
@@ -0,0 +1,232 @@
"""Every Ulysses path of USPAttention gives the same bytes with the exchange
pipelined over head groups as with the sequential exchange.
The pipeline only engages on a same-host Ulysses group with peer-to-peer access,
so nothing in the single-GPU suite reaches it. Runs on 2 ranks, and on 4 when
four GPUs are visible:
pytest -v python/sglang/multimodal_gen/test/single_test_file/test_usp_pipelined_attention_multi_gpu.py
"""
from __future__ import annotations
import os
import subprocess
import sys
import unittest
import torch
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.test.test_utils import CustomTestCase
def _worker() -> int:
import torch.distributed as dist
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
)
rank = int(os.environ["RANK"])
world = int(os.environ["WORLD_SIZE"])
torch.cuda.set_device(rank)
maybe_init_distributed_environment_and_model_parallel(
tp_size=1, sp_size=world, ulysses_degree=world
)
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
IPC_A2A,
)
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
IPC_A2A_MULTI,
)
from sglang.multimodal_gen.runtime.layers import usp
from sglang.multimodal_gen.runtime.layers.attention.layer import (
PIPELINED_ATTENTION_BACKENDS,
USPAttention,
)
from sglang.multimodal_gen.runtime.managers.forward_context import (
set_forward_context,
)
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
from sglang.multimodal_gen.test.unit.conftest import _make_unit_server_args
set_global_server_args(_make_unit_server_args())
peers = [d for d in range(world) if d != rank]
if not all(torch.cuda.can_device_access_peer(rank, d) for d in peers):
print("SKIP no peer-to-peer access between the devices", flush=True)
return 0
heads, kv_heads, dim, s_local, n_rep = 16, 8, 64, 96, 24
failures, ran = [], []
backends = [b for b in PIPELINED_ATTENTION_BACKENDS]
def shard(batch, rows, h, seed):
torch.manual_seed(seed) # identical full tensors on every rank
full = torch.randn(batch, rows * world, h, dim, device="cuda").bfloat16()
return full[:, rank * rows : (rank + 1) * rows].contiguous()
def replicated(batch, rows, h, seed):
torch.manual_seed(seed)
return torch.randn(batch, rows, h, dim, device="cuda").bfloat16()
def both(label, call):
"""`call()` sequentially and pipelined (two forced groups) must agree."""
outs = []
for groups in (0, 2):
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = groups
with torch.inference_mode(), set_forward_context(0, None, None):
outs.append(call())
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = -1
if not torch.equal(outs[0], outs[1]):
failures.append(label)
ran.append(label)
for backend in backends:
try:
layer = USPAttention(
num_heads=heads,
head_size=dim,
causal=False,
required_attention_backend=backend,
prefix=f"test.{backend.name}",
)
except Exception as e: # noqa: BLE001 - a backend this GPU cannot build
print(f"backend {backend.name} unavailable here: {e}", flush=True)
continue
if layer.backend is not backend:
print(
f"backend {backend.name} resolved to {layer.backend.name}", flush=True
)
continue
name = backend.name
for batch in (1, 2):
q, k, v = (shard(batch, s_local, heads, 10 + i) for i in range(3))
both(f"{name} dense b{batch}", lambda: layer(q, k, v))
rep = [replicated(batch, n_rep, heads, 20 + i) for i in range(3)]
joined = [torch.cat([r, t], dim=1) for r, t in zip(rep, (q, k, v))]
both(
f"{name} replicated prefix b{batch}",
lambda: layer(*joined, num_replicated_prefix=n_rep),
)
mask = torch.ones(batch, n_rep + s_local, dtype=torch.bool, device="cuda")
mask[:, n_rep // 2 : n_rep] = False # padded text tokens
both(
f"{name} replicated prefix + key mask b{batch}",
lambda: layer(*joined, num_replicated_prefix=n_rep, attn_mask=mask),
)
trailing = [torch.cat([t, r], dim=1) for r, t in zip(rep, (q, k, v))]
both(
f"{name} replicated suffix b{batch}",
lambda: layer(*trailing, num_replicated_suffix=n_rep),
)
k_rep, v_rep = (replicated(batch, n_rep, heads, 30 + i) for i in range(2))
both(
f"{name} replicated KV prefix b{batch}",
lambda: layer.forward_with_replicated_kv_prefix(q, k_rep, v_rep, k, v),
)
key_mask = torch.ones(batch, s_local, dtype=torch.bool, device="cuda")
key_mask[:, -s_local // 4 :] = rank % 2 == 0 # ragged across ranks
both(
f"{name} gathered key mask b{batch}",
lambda: layer(q, k, v, attn_mask=key_mask),
)
gqa = USPAttention(
num_heads=heads,
head_size=dim,
num_kv_heads=kv_heads,
causal=False,
required_attention_backend=backend,
prefix=f"test.gqa.{backend.name}",
)
q = shard(1, s_local, heads, 40)
k, v = (shard(1, s_local, kv_heads, 41 + i) for i in range(2))
both(f"{name} GQA dense", lambda: gqa(q, k, v))
# the first call of a shape checks the pipeline against a sequential
# exchange on per-call buffers: no staging sized for it stays behind
usp.drop_a2a_staging_buffers()
IPC_A2A.drop_staging()
checked = len(IPC_A2A_MULTI.verified)
q, k, v = (shard(1, 2 * s_local, heads, 50 + i) for i in range(3))
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = 2
with torch.inference_mode(), set_forward_context(0, None, None):
layer(q, k, v)
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = -1
if len(IPC_A2A_MULTI.verified) != checked + 1:
failures.append(f"{name}: a new shape was not checked")
if usp._A2A_STAGING_BUFFERS or IPC_A2A.staging:
failures.append(
f"{name}: the first-sight check left sequential staging behind "
f"({list(usp._A2A_STAGING_BUFFERS)}, {len(IPC_A2A.staging)} IPC pairs)"
)
# a comparison where every arm quietly fell back would pass while testing
# nothing: the pipeline must have run and verified every layout
if not ran:
failures.append("no pipelined backend could be built")
if IPC_A2A_MULTI.mismatched:
failures.append(
f"pipeline disagreed with the sequential exchange: {IPC_A2A_MULTI.mismatched}"
)
if len(IPC_A2A_MULTI.verified) < len({label.split(" b")[0] for label in ran}):
failures.append(
f"pipeline verified {len(IPC_A2A_MULTI.verified)} layouts for {len(ran)} cases"
)
verdict = torch.tensor([len(failures)], device="cuda")
dist.all_reduce(verdict)
if failures:
print(f"rank{rank} MISMATCH: {failures}", flush=True)
if rank == 0:
print(f"ran: {ran}", flush=True)
print(
f"USP_PIPELINE_PARITY {'FAIL' if verdict.item() else 'PASS'} world={world}",
flush=True,
)
dist.barrier()
dist.destroy_process_group()
return 1 if verdict.item() else 0
class TestUspPipelinedAttentionMultiGpu(CustomTestCase):
def _run(self, world: int, port: int):
if not current_platform.is_cuda():
self.skipTest("the copy-engine pipeline is CUDA-only")
if torch.cuda.device_count() < world:
self.skipTest(f"needs {world} GPUs")
proc = subprocess.run(
[
sys.executable,
"-m",
"torch.distributed.run",
f"--nproc-per-node={world}",
f"--master-port={port}",
__file__,
"--worker",
],
capture_output=True,
text=True,
timeout=1200,
)
print(proc.stdout[-4000:])
if proc.returncode != 0:
print(proc.stderr[-4000:], file=sys.stderr)
self.assertEqual(proc.returncode, 0, "pipelined USPAttention diverged")
if "SKIP" not in proc.stdout:
self.assertIn(f"USP_PIPELINE_PARITY PASS world={world}", proc.stdout)
def test_two_ranks_match_sequential_bitwise(self):
self._run(2, 29547)
def test_four_ranks_match_sequential_bitwise(self):
self._run(4, 29557)
if __name__ == "__main__":
if "--worker" in sys.argv:
raise SystemExit(_worker())
unittest.main()