[diffusion] perf: pipeline every USPAttention Ulysses path's exchange over the copy engine (#43521)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
416ebe6090
commit
72657f7ca5
@@ -68,10 +68,11 @@ if TYPE_CHECKING:
|
||||
# copy-engine all-to-all for Ulysses groups of any size on one host; off by
|
||||
# default while it is validated. Falls back to NCCL when unavailable.
|
||||
SGLANG_DIFFUSION_IPC_A2A_MULTI: bool = False
|
||||
# head groups for pipelining MiniMax-H3's Ulysses exchange against dense
|
||||
# attention over the copy-engine transport: -1 picks a count that divides
|
||||
# the heads per rank and steps aside when its buffers do not fit, 0 or 1
|
||||
# keeps the sequential exchange, N >= 2 forces N groups
|
||||
# head groups for pipelining the Ulysses exchange against dense attention
|
||||
# over the copy-engine transport (MiniMax-H3, and USPAttention's plain path):
|
||||
# -1 picks the most groups that divide the heads per rank while each group's
|
||||
# call still fills the GPU, and steps aside when the buffers do not fit; 0 or
|
||||
# 1 keeps the sequential exchange; N >= 2 forces N groups
|
||||
SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS: int = -1
|
||||
# a deadlock backstop, not a per-step budget: a rank can legitimately stall
|
||||
# for seconds (layerwise offload, wan2.2 expert-tower swaps), and expiry now
|
||||
|
||||
+522
-140
@@ -9,15 +9,19 @@ NCCL's all-to-all is an SM kernel, and next to a full-occupancy attention it
|
||||
slows that attention by more than the exchange costs.
|
||||
|
||||
Per peer, a sequence counter published after the copy orders the data before
|
||||
the flag, and the receiver spins on that peer's entry. Double-buffered slots
|
||||
alternate per call; a slot is only rewritten two calls later, after a spin on
|
||||
every peer, and each peer publishes that call only after it has consumed the
|
||||
slot in stream order.
|
||||
the flag, and the receiver spins on that peer's entry. The plain exchange's
|
||||
double-buffered slots alternate per call; a slot is only rewritten two calls
|
||||
later, after a spin on every peer, and each peer publishes that call only after
|
||||
it has consumed the slot in stream order. The pipelined attention needs one set
|
||||
of slots (see ``_PipelinePool``).
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import functools
|
||||
import logging
|
||||
import socket
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -32,8 +36,86 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# Group counts tried, in order, when the caller leaves the choice to the
|
||||
# transport: MiniMax-H3's 56 heads give 28, 14 and 7 per rank at Ulysses 2, 4
|
||||
# and 8, and 7 groups divide all three.
|
||||
_AUTO_PIPELINE_GROUPS = (7, 4, 2)
|
||||
# and 8, and 7 groups divide all three; Wan's 40 and 12 heads need 5 and 3.
|
||||
_AUTO_PIPELINE_GROUPS = (7, 5, 4, 3, 2)
|
||||
# Each group's attention call must fill this many waves of 128-row tiles: below
|
||||
# it the call's tail costs more than the overlap saves (cuDNN SDPA on B300 runs
|
||||
# Wan 1.3B at Ulysses 4 in three one-head groups 44% slower than sequentially).
|
||||
_MIN_GROUP_WAVES = 4
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _sm_count(device: int) -> int:
|
||||
return torch.cuda.get_device_properties(device).multi_processor_count
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PipelinePlan:
|
||||
"""Rows and heads of one pipelined attention call.
|
||||
|
||||
Each part (q, k, v) is a ``[batch, s_local, heads, head_dim]`` sequence
|
||||
shard plus ``replicated`` rows that every rank holds and that stay out of
|
||||
the exchange (text tokens), attended before the gathered sequence when
|
||||
``replicated_first`` and after it otherwise. A receive slot holds one part
|
||||
of one head group as attention reads it: its replicated rows and every
|
||||
rank's shard in rank order. The output keeps q's rows in this rank's
|
||||
layout: its replicated rows and this rank's shard.
|
||||
"""
|
||||
|
||||
batch: int
|
||||
s_local: int
|
||||
heads: tuple[int, int, int]
|
||||
head_dim: int
|
||||
replicated: tuple[int, int, int]
|
||||
replicated_first: bool
|
||||
world: int
|
||||
groups: int
|
||||
|
||||
def group_heads(self, part: int) -> int:
|
||||
return self.heads[part] // (self.world * self.groups)
|
||||
|
||||
def rows(self, part: int) -> int:
|
||||
return self.replicated[part] + self.world * self.s_local
|
||||
|
||||
def shard_row(self, part: int, rank: int) -> int:
|
||||
return (self.replicated[part] if self.replicated_first else 0) + (
|
||||
rank * self.s_local
|
||||
)
|
||||
|
||||
def replicated_row(self, part: int) -> int:
|
||||
return 0 if self.replicated_first else self.world * self.s_local
|
||||
|
||||
@property
|
||||
def out_rows(self) -> int:
|
||||
return self.replicated[0] + self.s_local
|
||||
|
||||
@property
|
||||
def out_shard_row(self) -> int:
|
||||
return self.replicated[0] if self.replicated_first else 0
|
||||
|
||||
@property
|
||||
def out_replicated_row(self) -> int:
|
||||
return 0 if self.replicated_first else self.s_local
|
||||
|
||||
def slot_numel(self, part: int) -> int:
|
||||
return self.batch * self.rows(part) * self.group_heads(part) * self.head_dim
|
||||
|
||||
def out_block_numel(self) -> int:
|
||||
return self.batch * self.out_rows * self.group_heads(0) * self.head_dim
|
||||
|
||||
|
||||
def _auto_pipeline_groups(plan: _PipelinePlan) -> int:
|
||||
"""The most head groups whose calls still fill the GPU, or 0 for none."""
|
||||
tiles = plan.batch * -(-plan.rows(0) // 128)
|
||||
floor = _MIN_GROUP_WAVES * _sm_count(torch.cuda.current_device())
|
||||
for n in _AUTO_PIPELINE_GROUPS:
|
||||
split = plan.world * n
|
||||
if (
|
||||
all(h % split == 0 for h in plan.heads)
|
||||
and plan.heads[0] // split * tiles >= floor
|
||||
):
|
||||
return n
|
||||
return 0
|
||||
|
||||
|
||||
def _member_devices(group, device: int) -> list[int]:
|
||||
@@ -114,15 +196,17 @@ class _StreamMemOps:
|
||||
raise _Unsupported(f"stream memory operations unavailable ({e})") from e
|
||||
stream.synchronize()
|
||||
|
||||
def write(self, stream, flag: torch.Tensor, value: int) -> None:
|
||||
def write(self, stream, flag: torch.Tensor | int, value: int) -> None:
|
||||
# default flags fence the stream's earlier writes (the copies) first
|
||||
rc = self._write(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
|
||||
ptr = flag if isinstance(flag, int) else flag.data_ptr()
|
||||
rc = self._write(stream.cuda_stream, ptr, value & 0xFFFFFFFF, 0)
|
||||
if rc:
|
||||
raise RuntimeError(f"cuStreamWriteValue32 failed with CUresult {rc}")
|
||||
|
||||
def wait(self, stream, flag: torch.Tensor, value: int) -> None:
|
||||
def wait(self, stream, flag: torch.Tensor | int, value: int) -> None:
|
||||
# CU_STREAM_WAIT_VALUE_GEQ compares cyclically, so counters may wrap
|
||||
rc = self._wait(stream.cuda_stream, flag.data_ptr(), value & 0xFFFFFFFF, 0)
|
||||
ptr = flag if isinstance(flag, int) else flag.data_ptr()
|
||||
rc = self._wait(stream.cuda_stream, ptr, value & 0xFFFFFFFF, 0)
|
||||
if rc:
|
||||
raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}")
|
||||
|
||||
@@ -153,41 +237,104 @@ class _StreamMemOps:
|
||||
raise RuntimeError(f"cuMemcpy2DAsync failed with CUresult {rc}")
|
||||
|
||||
|
||||
class _PipelineBuffers:
|
||||
"""Receive slots and flags for one pipelined-attention shape.
|
||||
class _PipelinePool:
|
||||
"""Receive slots, output blocks and flags that every pipelined call of one
|
||||
dtype shares, grown to the largest call seen rather than kept per shape.
|
||||
|
||||
Input slots are [2, groups, 3, world, chunk]: group g's q (k, v) rows from
|
||||
every rank form one contiguous [S, group_heads, head_dim] tensor for
|
||||
attention. Output slots are source-major ([2, world, groups, chunk]), each
|
||||
block a peer's rows for one head group, copied into the merged output as it
|
||||
arrives.
|
||||
One set serves every call, with no slot per call parity. Rank r sends a
|
||||
group's output blocks only after attending over its slot, and peer p
|
||||
returns from a call only after merging all of r's blocks; p writes r's
|
||||
slots for the next call only after that, so after r has read them. And p
|
||||
writes its next block into r's output block only after attending over r's
|
||||
rows of that call, which r sends only after its own previous call
|
||||
returned, so after r merged p's previous block.
|
||||
"""
|
||||
|
||||
def __init__(self, state, s_local, heads, head_dim, groups, dtype):
|
||||
def __init__(self, state, dtype, in_numel: int, out_numel: int, groups: int):
|
||||
world = state.world
|
||||
self.group_heads = heads // world // groups
|
||||
rows = s_local * self.group_heads
|
||||
zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda")
|
||||
self.inb = state._share(
|
||||
zeros(2, groups, 3, world, rows * head_dim), state.group
|
||||
)
|
||||
self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group)
|
||||
self.sizes = (in_numel, out_numel, groups)
|
||||
zeros = lambda n, dt=dtype: torch.zeros(n, dtype=dt, device="cuda")
|
||||
self.inb = state._share(zeros(in_numel), state.group)
|
||||
self.outb = state._share(zeros(out_numel), state.group)
|
||||
# fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g
|
||||
# peer p has delivered. Groups finish on separate streams, so a single
|
||||
# output counter could be lowered by a signal that lands late.
|
||||
self.fin = state._share(zeros(world, dt=torch.int32), state.group)
|
||||
self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group)
|
||||
# the caller's q/k for each peer block; v moves straight from its source
|
||||
self.send = torch.empty(
|
||||
world * groups,
|
||||
2,
|
||||
s_local,
|
||||
self.group_heads,
|
||||
head_dim,
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
)
|
||||
self.fout = state._share(zeros(groups * world, dt=torch.int32), state.group)
|
||||
self.inb_ptr = [t.data_ptr() for t in self.inb]
|
||||
self.outb_ptr = [t.data_ptr() for t in self.outb]
|
||||
self.fin_ptr = [t.data_ptr() for t in self.fin]
|
||||
self.fout_ptr = [t.data_ptr() for t in self.fout]
|
||||
self.calls = 0
|
||||
self.groups_sent = 0
|
||||
# the caller's q/k for each peer block when a fill writes them; v moves
|
||||
# straight from its source
|
||||
self.send = None
|
||||
|
||||
def nbytes(self, esz: int) -> int:
|
||||
return (self.sizes[0] + self.sizes[1]) * esz
|
||||
|
||||
|
||||
class _PipelineSlots:
|
||||
"""Where one plan's receive slots and output blocks sit in the pool.
|
||||
|
||||
Receive slots, per head group and part: one contiguous ``[batch, rows,
|
||||
group_heads, head_dim]`` tensor laid out as attention reads it (see
|
||||
``_PipelinePlan``). Output blocks are source-major: block ``(source,
|
||||
group)`` is that rank's output rows for this rank, one ``[batch, out_rows,
|
||||
group_heads, head_dim]`` block in the output's row order, copied into the
|
||||
merged output as it arrives.
|
||||
"""
|
||||
|
||||
def __init__(self, plan: _PipelinePlan, dtype):
|
||||
self.plan = plan
|
||||
self.dtype = dtype
|
||||
numel = [plan.slot_numel(part) for part in range(3)]
|
||||
self.part_numel = numel
|
||||
self.part_offset = (0, numel[0], numel[0] + numel[1])
|
||||
self.group_numel = sum(numel)
|
||||
self.out_numel = plan.out_block_numel()
|
||||
self.in_total = plan.groups * self.group_numel
|
||||
self.out_total = plan.world * plan.groups * self.out_numel
|
||||
# raw addresses: the copies are issued per call from plain integers,
|
||||
# since a tensor view per copy costs more host time than the copy
|
||||
self.esz = torch.empty((), dtype=dtype).element_size()
|
||||
self.rows = [plan.rows(part) for part in range(3)]
|
||||
self.row_bytes = [
|
||||
plan.group_heads(part) * plan.head_dim * self.esz for part in range(3)
|
||||
]
|
||||
self.send_numel = 2 * plan.s_local * plan.group_heads(0) * plan.head_dim
|
||||
|
||||
def slot_ptr(self, pool, member, group, part, batch_row, row) -> int:
|
||||
elems = group * self.group_numel + self.part_offset[part]
|
||||
return (
|
||||
pool.inb_ptr[member]
|
||||
+ elems * self.esz
|
||||
+ (batch_row * self.rows[part] + row) * self.row_bytes[part]
|
||||
)
|
||||
|
||||
def out_block_ptr(self, pool, member, source, group) -> int:
|
||||
block = (source * self.plan.groups + group) * self.out_numel
|
||||
return pool.outb_ptr[member] + block * self.esz
|
||||
|
||||
def slot(self, pool, member: int, group: int, part: int) -> torch.Tensor:
|
||||
"""``member``'s receive slot, mapped in this rank's context."""
|
||||
plan = self.plan
|
||||
start = group * self.group_numel + self.part_offset[part]
|
||||
return (
|
||||
pool.inb[member]
|
||||
.narrow(0, start, self.part_numel[part])
|
||||
.view(plan.batch, plan.rows(part), plan.group_heads(part), plan.head_dim)
|
||||
)
|
||||
|
||||
def send_block(self, pool, peer: int, group: int) -> torch.Tensor:
|
||||
plan = self.plan
|
||||
need = plan.world * plan.groups * self.send_numel
|
||||
if pool.send is None or pool.send.numel() < need:
|
||||
pool.send = torch.empty(need, dtype=self.dtype, device="cuda")
|
||||
return pool.send.narrow(
|
||||
0, (peer * plan.groups + group) * self.send_numel, self.send_numel
|
||||
).view(2, plan.s_local, plan.group_heads(0), plan.head_dim)
|
||||
|
||||
|
||||
class IpcA2AMultiState:
|
||||
@@ -213,17 +360,29 @@ class IpcA2AMultiState:
|
||||
self.pipe_group_streams = []
|
||||
# pipeline shapes whose buffers did not fit on some rank; agreed by all
|
||||
self.declined = set()
|
||||
# (shape, caller signature) checked against the sequential exchange,
|
||||
# by outcome; agreed by all
|
||||
self.verified = set()
|
||||
self.mismatched = set()
|
||||
# call shapes -> the plan they resolve to, None when they cannot
|
||||
# pipeline; repeated calls skip straight to the answer
|
||||
self.plans = {}
|
||||
# dtype -> the _PipelinePool every pipelined call of that dtype uses
|
||||
self.pools = {}
|
||||
# ("pipeline", plan, dtype) -> its _PipelineSlots
|
||||
self.slots = {}
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop mappings that belong to a model-parallel group being replaced."""
|
||||
self.__init__()
|
||||
|
||||
def drop_staging(self) -> None:
|
||||
if not self.staging:
|
||||
if not self.staging and not self.pools:
|
||||
return
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.synchronize()
|
||||
self.staging.clear()
|
||||
self.pools.clear()
|
||||
|
||||
def _share(self, t: torch.Tensor, group) -> list[torch.Tensor]:
|
||||
"""Every member's `t`, peers' re-opened in the LOCAL device context
|
||||
@@ -314,53 +473,112 @@ class IpcA2AMultiState:
|
||||
self.staging.popitem(last=False)
|
||||
self.staging[key] = value
|
||||
|
||||
def pipelined_attention(self, q, k, v, attend, groups: int, fill=None):
|
||||
def pipelined_attention(
|
||||
self,
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attend,
|
||||
groups: int,
|
||||
fill=None,
|
||||
sequential=None,
|
||||
signature=None,
|
||||
replicated=(None, None, None),
|
||||
replicated_first: bool = True,
|
||||
):
|
||||
"""Ulysses attention with the head exchange pipelined against it.
|
||||
|
||||
q/k/v are [s_local, heads, head_dim], sequence-sharded with every head.
|
||||
``attend(q, k, v)`` runs attention on [S, heads / (world * groups),
|
||||
head_dim] tensors of one head group and must treat heads independently,
|
||||
which keeps the result bit-identical to the sequential exchange.
|
||||
Group g's copies run on the copy engine while group g-1 attends, and
|
||||
each group attends on its own stream and sends its output from there,
|
||||
so no stream waits on a kernel still running on another. Every copy,
|
||||
the merge into the output included, is a pitched copy-engine copy, so
|
||||
the exchange takes no SMs. Returns [s_local, heads, head_dim], or None
|
||||
when this call cannot pipeline. ``groups < 0`` picks the count and
|
||||
steps aside, on every rank, when the buffers would not fit.
|
||||
q/k/v are ``[batch, s_local, heads, head_dim]`` sequence shards with
|
||||
every head (``[s_local, heads, head_dim]`` reads as batch 1 and returns
|
||||
that shape); k and v may have fewer heads than q (GQA).
|
||||
``replicated`` optionally holds, per part, ``[batch, n, heads,
|
||||
head_dim]`` rows every rank holds (text tokens); they stay out of the
|
||||
exchange, attend before the gathered sequence when
|
||||
``replicated_first`` and after it otherwise, and their output, all
|
||||
heads, leads (or trails) the returned rows.
|
||||
|
||||
``fill(head_start, head_count, q_dst, k_dst)``, when given, writes those
|
||||
heads' q and k into two [s_local, head_count, head_dim] blocks in place
|
||||
of a copy (the caller's QK-norm writes its output there); blocks for
|
||||
this rank go straight into its own receive slot, and v always moves
|
||||
straight from ``v``. It is only called once the call is known to
|
||||
pipeline.
|
||||
``attend(q, k, v)`` runs attention on one head group, ``[batch, rows,
|
||||
heads / (world * groups), head_dim]`` per part in the sequence order
|
||||
above, and must treat heads independently, which keeps the result
|
||||
bit-identical to the sequential exchange. Group g's copies run on the
|
||||
copy engine while group g-1 attends, and each group attends on its own
|
||||
stream and sends its output from there, so no stream waits on a kernel
|
||||
still running on another. Every copy, the merge into the output
|
||||
included, is a pitched copy-engine copy, so the exchange takes no SMs.
|
||||
Returns the output rows of this rank, or None when this call cannot
|
||||
pipeline. ``groups < 0`` picks the count and steps aside, on every
|
||||
rank, when the buffers would not fit.
|
||||
|
||||
``fill(head_start, head_count, q_dst, k_dst)``, when given (batch 1,
|
||||
no replicated rows, as many k as q heads), writes those heads' q and k
|
||||
into two [s_local, head_count, head_dim] blocks in place of a copy
|
||||
(the caller's QK-norm writes its output there); blocks for this rank
|
||||
go straight into its own receive slot, and v always moves straight
|
||||
from ``v``. It is only called once the call is known to pipeline.
|
||||
|
||||
``sequential()``, when given, returns what the sequential exchange
|
||||
gives for this call. The first call per plan and ``signature`` (the
|
||||
caller's attention setup) runs both and keeps the pipeline for them
|
||||
only if every rank sees the same bytes, so a backend that does not
|
||||
treat a head group as it treats those heads in the full call never
|
||||
changes a result. It runs the sequential exchange first, with the
|
||||
pool released, so that call never holds both at once.
|
||||
"""
|
||||
world, r = self.world, self.rank
|
||||
s_local, heads, head_dim = q.shape
|
||||
squeeze = q.dim() == 3
|
||||
if squeeze:
|
||||
q, k, v = q[None], k[None], v[None]
|
||||
parts = (q, k, v)
|
||||
replicated = tuple(replicated)
|
||||
batch, s_local, _, head_dim = q.shape
|
||||
auto = groups < 0
|
||||
if auto:
|
||||
groups = next(
|
||||
(n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0
|
||||
shapes = (
|
||||
q.shape,
|
||||
k.shape,
|
||||
v.shape,
|
||||
tuple(None if t is None else t.shape for t in replicated),
|
||||
replicated_first,
|
||||
groups,
|
||||
fill is not None,
|
||||
)
|
||||
if shapes not in self.plans:
|
||||
self.plans[shapes] = self._plan(
|
||||
q, k, v, replicated, replicated_first, groups, fill is not None
|
||||
)
|
||||
plan = self.plans[shapes]
|
||||
if plan is None:
|
||||
return None
|
||||
groups = plan.groups
|
||||
|
||||
# each token's heads of one group must be one contiguous run to move
|
||||
# with a pitched copy
|
||||
copied = (v,) if fill is not None else (q, k, v)
|
||||
if (
|
||||
not groups
|
||||
or heads % (world * groups)
|
||||
or not (q.shape == k.shape == v.shape)
|
||||
or any(t.stride(2) != 1 or t.stride(1) != head_dim for t in copied)
|
||||
):
|
||||
copied = [v] if fill is not None else [q, k, v]
|
||||
copied += [t for t in replicated if t is not None]
|
||||
if not all(t.stride(-1) == 1 and t.stride(-2) == head_dim for t in copied):
|
||||
return None
|
||||
key = ("pipeline", s_local, heads, head_dim, groups, q.dtype)
|
||||
bufs = self.staging.get(key)
|
||||
if bufs is None:
|
||||
key = ("pipeline", plan, q.dtype)
|
||||
check = None if sequential is None else (key, signature)
|
||||
if check in self.mismatched:
|
||||
return None
|
||||
if check in self.verified:
|
||||
check = None
|
||||
slots = self.slots.get(key)
|
||||
if slots is None:
|
||||
slots = self.slots[key] = _PipelineSlots(plan, q.dtype)
|
||||
pool = self.pools.get(q.dtype)
|
||||
sizes = (slots.in_total, slots.out_total, groups)
|
||||
if pool is not None:
|
||||
sizes = tuple(map(max, sizes, pool.sizes))
|
||||
reference = None
|
||||
if pool is None or pool.sizes != sizes or check is not None:
|
||||
if torch.cuda.is_current_stream_capturing() or key in self.declined:
|
||||
return None
|
||||
# send + both receive and output slots: 10 * s_local * heads * head_dim
|
||||
need = 10 * s_local * heads * head_dim * q.element_size()
|
||||
if auto and not self._fits_on_every_rank(need):
|
||||
esz = q.element_size()
|
||||
held = 0 if pool is None else pool.nbytes(esz)
|
||||
need = (sizes[0] + sizes[1]) * esz
|
||||
if fill is not None:
|
||||
need += plan.world * groups * slots.send_numel * esz
|
||||
if auto and not self._fits_on_every_rank(need, reclaimable=held):
|
||||
logger.info(
|
||||
"Ulysses pipeline buffers (%.1f GiB) do not fit; using the "
|
||||
"sequential exchange for this shape",
|
||||
@@ -368,82 +586,137 @@ class IpcA2AMultiState:
|
||||
)
|
||||
self.declined.add(key)
|
||||
return None
|
||||
bufs = _PipelineBuffers(self, s_local, heads, head_dim, groups, q.dtype)
|
||||
self._insert(key, bufs)
|
||||
if pool is not None:
|
||||
# queued copies may still target the old mappings
|
||||
torch.cuda.synchronize()
|
||||
del self.pools[q.dtype]
|
||||
pool = None
|
||||
if check is not None:
|
||||
reference = sequential()
|
||||
pool = self.pools[q.dtype] = _PipelinePool(self, q.dtype, *sizes)
|
||||
if self.pipe_in is None:
|
||||
self.pipe_in = torch.cuda.Stream()
|
||||
while len(self.pipe_group_streams) < groups:
|
||||
self.pipe_group_streams.append(torch.cuda.Stream())
|
||||
mem, cin, hg = self.memops, self.pipe_in, bufs.group_heads
|
||||
mem, cin = self.memops, self.pipe_in
|
||||
main = torch.cuda.current_stream()
|
||||
slot = bufs.calls % 2
|
||||
base = bufs.calls * groups
|
||||
bufs.calls += 1
|
||||
call = bufs.calls
|
||||
base = pool.groups_sent
|
||||
pool.groups_sent += groups
|
||||
pool.calls += 1
|
||||
call = pool.calls
|
||||
peers = [(r + step) % world for step in range(1, world)]
|
||||
esz = q.element_size()
|
||||
width = hg * head_dim * esz # one token's heads of one group
|
||||
chunk = s_local * width # one rank's rows of one group, one of q/k/v
|
||||
start = lambda p, g: (p * groups + g) * hg # first head of block (p, g)
|
||||
slot_of = lambda p, g, part: bufs.inb[p][slot, g, part, r] # my rows there
|
||||
head_bytes = head_dim * esz
|
||||
row_bytes = slots.row_bytes
|
||||
# first head of block (p, g) of a part
|
||||
start = lambda p, g, part: (p * groups + g) * plan.group_heads(part)
|
||||
# (address, batch pitch, token pitch) of each part's shard and replicated rows
|
||||
src = [(t.data_ptr(), t.stride(0) * esz, t.stride(1) * esz) for t in parts]
|
||||
rep_src = [
|
||||
None if t is None else (t.data_ptr(), t.stride(0) * esz, t.stride(1) * esz)
|
||||
for t in replicated
|
||||
]
|
||||
my_shard_row = [plan.shard_row(part, r) for part in range(3)]
|
||||
|
||||
def send_rows(stream, t, p, g, part):
|
||||
src = t[:, start(p, g)]
|
||||
mem.copy2d(
|
||||
stream,
|
||||
slot_of(p, g, part).data_ptr(),
|
||||
width,
|
||||
src.data_ptr(),
|
||||
t.stride(0) * esz,
|
||||
width,
|
||||
s_local,
|
||||
)
|
||||
def send_rows(stream, part, p, g):
|
||||
# my shard of one part and head group into rank p's receive slot
|
||||
ptr, batch_pitch, token_pitch = src[part]
|
||||
ptr += start(p, g, part) * head_bytes
|
||||
for b in range(batch):
|
||||
mem.copy2d(
|
||||
stream,
|
||||
slots.slot_ptr(pool, p, g, part, b, my_shard_row[part]),
|
||||
row_bytes[part],
|
||||
ptr + b * batch_pitch,
|
||||
token_pitch,
|
||||
row_bytes[part],
|
||||
s_local,
|
||||
)
|
||||
|
||||
def place_replicated(stream, part, g):
|
||||
# rows every rank holds go straight into my own slot
|
||||
n = plan.replicated[part]
|
||||
if not n:
|
||||
return
|
||||
ptr, batch_pitch, token_pitch = rep_src[part]
|
||||
ptr += start(r, g, part) * head_bytes
|
||||
row = plan.replicated_row(part)
|
||||
for b in range(batch):
|
||||
mem.copy2d(
|
||||
stream,
|
||||
slots.slot_ptr(pool, r, g, part, b, row),
|
||||
row_bytes[part],
|
||||
ptr + b * batch_pitch,
|
||||
token_pitch,
|
||||
row_bytes[part],
|
||||
n,
|
||||
)
|
||||
|
||||
ready = main.record_event()
|
||||
own, sent = [], []
|
||||
if fill is not None:
|
||||
# group by group: the peers' blocks first, so their copies start
|
||||
# while this rank fills its own block straight into its slot
|
||||
block = lambda t: t.view(s_local, hg, head_dim)
|
||||
own_rows = lambda part, g: slots.slot(pool, r, g, part)[
|
||||
0, my_shard_row[part] : my_shard_row[part] + s_local
|
||||
]
|
||||
hg = plan.group_heads(0)
|
||||
for g in range(groups):
|
||||
for p in peers:
|
||||
qk = bufs.send[p * groups + g]
|
||||
fill(start(p, g), hg, qk[0], qk[1])
|
||||
qk = slots.send_block(pool, p, g)
|
||||
fill(start(p, g, 0), hg, qk[0], qk[1])
|
||||
sent.append(main.record_event())
|
||||
fill(start(r, g), hg, block(slot_of(r, g, 0)), block(slot_of(r, g, 1)))
|
||||
fill(start(r, g, 0), hg, own_rows(0, g), own_rows(1, g))
|
||||
own.append(main.record_event())
|
||||
own_cin = []
|
||||
cin.wait_event(ready)
|
||||
with torch.cuda.stream(cin):
|
||||
for g in range(groups):
|
||||
for p in (r, *peers):
|
||||
send_rows(cin, v, p, g, 2)
|
||||
send_rows(cin, 2, p, g)
|
||||
if fill is None:
|
||||
send_rows(cin, q, p, g, 0)
|
||||
send_rows(cin, k, p, g, 1)
|
||||
send_rows(cin, 0, p, g)
|
||||
send_rows(cin, 1, p, g)
|
||||
if p == r:
|
||||
for part in range(3):
|
||||
place_replicated(cin, part, g)
|
||||
own_cin.append(cin.record_event())
|
||||
if fill is not None:
|
||||
cin.wait_event(sent[g])
|
||||
chunk = s_local * row_bytes[0]
|
||||
for p in peers:
|
||||
q_slot = slots.slot_ptr(pool, p, g, 0, 0, my_shard_row[0])
|
||||
k_slot = slots.slot_ptr(pool, p, g, 1, 0, my_shard_row[1])
|
||||
# q then k: two rows of `chunk`, one part apart in the slot
|
||||
mem.copy2d(
|
||||
cin,
|
||||
slot_of(p, g, 0).data_ptr(),
|
||||
slot_of(p, g, 1).data_ptr() - slot_of(p, g, 0).data_ptr(),
|
||||
bufs.send[p * groups + g].data_ptr(),
|
||||
q_slot,
|
||||
k_slot - q_slot,
|
||||
slots.send_block(pool, p, g).data_ptr(),
|
||||
chunk,
|
||||
chunk,
|
||||
2,
|
||||
)
|
||||
for p in peers:
|
||||
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
|
||||
mem.write(cin, pool.fin_ptr[p] + 4 * r, base + g + 1)
|
||||
# Head block b = p * groups + g of the output is group g of rank p's
|
||||
# heads. Each group stream writes its blocks as soon as they exist, so
|
||||
# the merge overlaps the groups still attending instead of trailing them.
|
||||
merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device)
|
||||
merged_blocks = merged.view(s_local, world * groups, hg, head_dim)
|
||||
merged_pitch = heads * head_dim * esz
|
||||
hq = plan.group_heads(0)
|
||||
merged = torch.empty(
|
||||
batch,
|
||||
plan.out_rows,
|
||||
plan.heads[0],
|
||||
head_dim,
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
merged_ptr = merged.data_ptr()
|
||||
merged_row = plan.heads[0] * head_bytes
|
||||
out_row = row_bytes[0]
|
||||
block_batch = plan.out_rows * out_row
|
||||
shard_out, rep_out = plan.out_shard_row, plan.out_replicated_row
|
||||
n_rep, rep_row = plan.replicated[0], plan.replicated_row(0)
|
||||
done = []
|
||||
for g in range(groups):
|
||||
stream = self.pipe_group_streams[g]
|
||||
@@ -452,37 +725,60 @@ class IpcA2AMultiState:
|
||||
stream.wait_event(own_cin[g])
|
||||
with torch.cuda.stream(stream):
|
||||
for p in peers:
|
||||
mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1)
|
||||
part = lambda i: bufs.inb[r][slot, g, i].view(
|
||||
world * s_local, hg, head_dim
|
||||
)
|
||||
out = attend(part(0), part(1), part(2)).contiguous()
|
||||
rows = lambda p: out[p * s_local : (p + 1) * s_local]
|
||||
mem.wait(stream, pool.fin_ptr[r] + 4 * p, base + g + 1)
|
||||
views = [slots.slot(pool, r, g, part) for part in range(3)]
|
||||
if squeeze:
|
||||
out = attend(*(t[0] for t in views)).contiguous()[None]
|
||||
else:
|
||||
out = attend(*views).contiguous()
|
||||
out_ptr = out.data_ptr()
|
||||
out_batch = out.stride(0) * esz
|
||||
for p in peers:
|
||||
bufs.outb[p][slot, r, g].copy_(
|
||||
rows(p).reshape(-1), non_blocking=True
|
||||
)
|
||||
# this peer's rows and the replicated rows every rank needs,
|
||||
# one contiguous run per batch row
|
||||
block = slots.out_block_ptr(pool, p, r, g)
|
||||
runs = [(shard_out, plan.shard_row(0, p), s_local)]
|
||||
if n_rep:
|
||||
runs.append((rep_out, rep_row, n_rep))
|
||||
for dst_row, src_row, n in runs:
|
||||
mem.copy2d(
|
||||
stream,
|
||||
block + dst_row * out_row,
|
||||
block_batch,
|
||||
out_ptr + src_row * out_row,
|
||||
out_batch,
|
||||
n * out_row,
|
||||
batch,
|
||||
)
|
||||
for p in peers:
|
||||
mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call)
|
||||
mem.copy2d(
|
||||
stream,
|
||||
merged_blocks[:, r * groups + g].data_ptr(),
|
||||
merged_pitch,
|
||||
rows(r).data_ptr(),
|
||||
width,
|
||||
width,
|
||||
s_local,
|
||||
)
|
||||
mem.write(stream, pool.fout_ptr[p] + 4 * (g * world + r), call)
|
||||
own_head = merged_ptr + start(r, g, 0) * head_bytes
|
||||
for b in range(batch):
|
||||
runs = [(shard_out, my_shard_row[0], s_local)]
|
||||
if n_rep:
|
||||
runs.append((rep_out, rep_row, n_rep))
|
||||
for dst_row, src_row, n in runs:
|
||||
mem.copy2d(
|
||||
stream,
|
||||
own_head + (b * plan.out_rows + dst_row) * merged_row,
|
||||
merged_row,
|
||||
out_ptr + b * out_batch + src_row * out_row,
|
||||
out_row,
|
||||
out_row,
|
||||
n,
|
||||
)
|
||||
for p in peers:
|
||||
mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call)
|
||||
mem.wait(stream, pool.fout_ptr[r] + 4 * (g * world + p), call)
|
||||
# the block and the merged output both have one row pitch
|
||||
# across the batch, so a single copy covers every batch row
|
||||
mem.copy2d(
|
||||
stream,
|
||||
merged_blocks[:, p * groups + g].data_ptr(),
|
||||
merged_pitch,
|
||||
bufs.outb[r][slot, p, g].data_ptr(),
|
||||
width,
|
||||
width,
|
||||
s_local,
|
||||
merged_ptr + start(p, g, 0) * head_bytes,
|
||||
merged_row,
|
||||
slots.out_block_ptr(pool, r, p, g),
|
||||
out_row,
|
||||
out_row,
|
||||
batch * plan.out_rows,
|
||||
)
|
||||
done.append(stream.record_event())
|
||||
for event in done:
|
||||
@@ -490,15 +786,78 @@ class IpcA2AMultiState:
|
||||
# the next call's fill rewrites `send` and the caller may free `v`, both
|
||||
# read by the copy stream
|
||||
main.wait_stream(cin)
|
||||
if squeeze:
|
||||
merged = merged[0]
|
||||
if check is not None:
|
||||
return self._first_sight(check, merged, reference)
|
||||
return merged
|
||||
|
||||
def _fits_on_every_rank(self, nbytes: int) -> bool:
|
||||
def _plan(self, q, k, v, replicated, replicated_first, groups, filled):
|
||||
"""The plan these shapes pipeline with, or None; rank-independent."""
|
||||
world = self.world
|
||||
batch, s_local, _, head_dim = q.shape
|
||||
parts = (q, k, v)
|
||||
rows = tuple(0 if t is None else t.shape[1] for t in replicated)
|
||||
if (
|
||||
k.shape != v.shape
|
||||
or any(
|
||||
t.shape[:2] != (batch, s_local) or t.shape[3] != head_dim for t in parts
|
||||
)
|
||||
or rows[1] != rows[2]
|
||||
or any(
|
||||
t is not None and t.shape != (batch, n, parts[i].shape[2], head_dim)
|
||||
for i, (t, n) in enumerate(zip(replicated, rows))
|
||||
)
|
||||
or (filled and (batch != 1 or any(rows) or q.shape != k.shape))
|
||||
):
|
||||
return None
|
||||
plan = _PipelinePlan(
|
||||
batch=batch,
|
||||
s_local=s_local,
|
||||
heads=tuple(t.shape[2] for t in parts),
|
||||
head_dim=head_dim,
|
||||
replicated=rows,
|
||||
replicated_first=replicated_first,
|
||||
world=world,
|
||||
groups=groups,
|
||||
)
|
||||
if groups < 0:
|
||||
plan = dataclasses.replace(plan, groups=_auto_pipeline_groups(plan))
|
||||
if not plan.groups or any(h % (world * plan.groups) for h in plan.heads):
|
||||
return None
|
||||
return plan
|
||||
|
||||
def _first_sight(self, check, merged, reference):
|
||||
"""Keep the pipeline for `check` only if it matches the sequential
|
||||
exchange's `reference` byte for byte on every rank; a group collective."""
|
||||
same = torch.tensor(
|
||||
[int(torch.equal(merged, reference))], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
dist.all_reduce(same, op=dist.ReduceOp.MIN, group=self.group)
|
||||
if same.item():
|
||||
self.verified.add(check)
|
||||
return merged
|
||||
self.mismatched.add(check)
|
||||
plan = check[0][1]
|
||||
logger.info(
|
||||
"Ulysses pipeline over %d head groups differs from the sequential "
|
||||
"exchange for %s under %s; keeping the sequential exchange",
|
||||
plan.groups,
|
||||
plan,
|
||||
check[1],
|
||||
)
|
||||
return reference
|
||||
|
||||
def _fits_on_every_rank(self, nbytes: int, reclaimable: int = 0) -> bool:
|
||||
"""Whether every rank can spare twice `nbytes`, counting the allocator's
|
||||
cached blocks; a group collective, so all ranks reach the same answer."""
|
||||
cached blocks and `reclaimable` bytes it is about to free; a group
|
||||
collective, so all ranks reach the same answer."""
|
||||
free, _ = torch.cuda.mem_get_info()
|
||||
cached = torch.cuda.memory_reserved() - torch.cuda.memory_allocated()
|
||||
fits = torch.tensor(
|
||||
[int(free + cached >= 2 * nbytes)], dtype=torch.int32, device="cuda"
|
||||
[int(free + cached + reclaimable >= 2 * nbytes)],
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
dist.all_reduce(fits, op=dist.ReduceOp.MIN, group=self.group)
|
||||
return bool(fits.item())
|
||||
@@ -598,12 +957,35 @@ def ipc_a2a_multi_ready(group, enabled: bool | None = None) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def ulysses_pipelined_attention(q, k, v, attend, groups: int, fill=None):
|
||||
def ulysses_pipelined_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attend,
|
||||
groups: int,
|
||||
fill=None,
|
||||
sequential=None,
|
||||
signature=None,
|
||||
replicated=(None, None, None),
|
||||
replicated_first: bool = True,
|
||||
):
|
||||
"""Pipelined Ulysses attention over the copy-engine transport, or None
|
||||
when the Ulysses group cannot use it (the caller keeps its NCCL path)."""
|
||||
when the Ulysses group cannot use it (the caller keeps its NCCL path).
|
||||
See ``IpcA2AMultiState.pipelined_attention`` for the arguments."""
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_sp_group
|
||||
|
||||
group = get_sp_group().ulysses_group
|
||||
if group is None or not ipc_a2a_multi_ready(group, enabled=True):
|
||||
return None
|
||||
return IPC_A2A_MULTI.pipelined_attention(q, k, v, attend, groups, fill=fill)
|
||||
return IPC_A2A_MULTI.pipelined_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attend,
|
||||
groups,
|
||||
fill=fill,
|
||||
sequential=sequential,
|
||||
signature=signature,
|
||||
replicated=replicated,
|
||||
replicated_first=replicated_first,
|
||||
)
|
||||
|
||||
@@ -68,6 +68,7 @@ from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_output_all_to_all,
|
||||
_usp_output_all_to_all_varlen,
|
||||
ring_attn,
|
||||
uncached_a2a_staging,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
ForwardContext,
|
||||
@@ -95,6 +96,14 @@ _VARLEN_FA_ENABLED = os.environ.get("SGLANG_VARLEN_FA", "1") != "0"
|
||||
# and run dense attention on the padded layout.
|
||||
_SP_PAD_MASK_DISABLED = envs.SGLANG_DIFFUSION_DISABLE_SP_PAD_MASK
|
||||
|
||||
# Dense backends the head-group pipeline runs against: each computes a head
|
||||
# independently, so a head group comes out as those heads do in the full call
|
||||
# (FlashAttention on Hopper, cuDNN SDPA on Blackwell).
|
||||
PIPELINED_ATTENTION_BACKENDS = (
|
||||
AttentionBackendEnum.FA,
|
||||
AttentionBackendEnum.DYNAMIC_CUDNN_SDPA,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_sp_attention_mode(
|
||||
*, causal: bool, sparse_backend: bool
|
||||
@@ -890,6 +899,7 @@ class USPAttention(nn.Module):
|
||||
|
||||
self.skip_sequence_parallel = skip_sequence_parallel
|
||||
self.enable_packed_qkv_input_a2a = bool(enable_packed_qkv_input_a2a)
|
||||
self._pipeline_declined: set = set()
|
||||
self.sp_attention_mode, self.sp_attention_mode_is_auto = (
|
||||
_resolve_sp_attention_mode(
|
||||
causal=causal, sparse_backend=self.backend.is_sparse
|
||||
@@ -1259,14 +1269,10 @@ class USPAttention(nn.Module):
|
||||
)
|
||||
|
||||
sp_size = get_ulysses_parallel_world_size()
|
||||
if sp_size > 1 and not qkv_pre_all_to_all:
|
||||
qkv_fast = _ipc_input_a2a_qkv(q, k, v)
|
||||
if qkv_fast is not None:
|
||||
q, k, v = qkv_fast
|
||||
else:
|
||||
q, k, v = _usp_input_all_to_all_qkv(q, k, v)
|
||||
|
||||
if (
|
||||
exchange = sp_size > 1 and not qkv_pre_all_to_all
|
||||
# the gathered sequence the attention below runs over
|
||||
gathered_seq = q.shape[1] * (sp_size if exchange else 1)
|
||||
tail_fa = (
|
||||
_VARLEN_FA_ENABLED
|
||||
and self.backend == AttentionBackendEnum.FA
|
||||
and meta_pad_start is not None
|
||||
@@ -1274,109 +1280,121 @@ class USPAttention(nn.Module):
|
||||
and meta_pad_end > meta_pad_start
|
||||
and q.device.type == "cuda"
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
):
|
||||
bs, seq = q.shape[0], q.shape[1]
|
||||
assert 0 <= meta_pad_start < meta_pad_end <= seq
|
||||
cu_tail = attn_mask_meta.get("cu_seqlens_tail")
|
||||
if cu_tail is not None and meta_pad_end == seq:
|
||||
# Zero-copy tail path: run varlen FA straight over the
|
||||
# padded layout, each row split into [valid | pad] segments
|
||||
# (contiguous reshapes only, no repacking).
|
||||
assert cu_tail.numel() == 2 * bs + 1, (
|
||||
"cu_seqlens_tail does not match the batch size"
|
||||
)
|
||||
gathered_mask = gathered_mask_meta = None
|
||||
if not tail_fa:
|
||||
# If NCCL timeout/deadlock occurs here, check whether
|
||||
# attn_mask is inconsistent across SP ranks (None on some,
|
||||
# Tensor on others), which causes all_gather participant
|
||||
# mismatch. Upstream mask builders must ensure all ranks
|
||||
# produce the same mask type.
|
||||
if attn_mask is None:
|
||||
# Meta-only tail-pad caller on a non-FA fallback: the
|
||||
# gathered mask is fully determined by the pad span, no
|
||||
# collective needed.
|
||||
gathered_mask = torch.ones(
|
||||
q.shape[0], gathered_seq, dtype=torch.bool, device=q.device
|
||||
)
|
||||
out = flash_attn_varlen_func(
|
||||
q=q.reshape(bs * seq, *q.shape[2:]),
|
||||
k=k.reshape(bs * seq, *k.shape[2:]),
|
||||
v=v.reshape(bs * seq, *v.shape[2:]),
|
||||
cu_seqlens_q=cu_tail,
|
||||
cu_seqlens_k=cu_tail,
|
||||
max_seqlen_q=attn_mask_meta["max_seqlen_tail"],
|
||||
max_seqlen_k=attn_mask_meta["max_seqlen_tail"],
|
||||
gathered_mask[:, meta_pad_start:meta_pad_end] = False
|
||||
else:
|
||||
gathered_mask = sequence_model_parallel_all_gather(
|
||||
attn_mask.contiguous(), dim=1
|
||||
)
|
||||
gathered_rows = (q.shape[0], gathered_seq)
|
||||
if (
|
||||
_VARLEN_FA_ENABLED
|
||||
and self.backend == AttentionBackendEnum.FA
|
||||
and gathered_mask.dtype
|
||||
in (torch.bool, torch.uint8, torch.int32, torch.int64)
|
||||
and q.device.type == "cuda"
|
||||
and gathered_mask.device == q.device
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
and gathered_rows == gathered_mask.shape
|
||||
and k.shape[:2] == v.shape[:2] == q.shape[:2]
|
||||
):
|
||||
gathered_mask_meta = build_varlen_mask_meta(gathered_mask)
|
||||
assert gathered_mask_meta["inv_indices"].shape[0] == (
|
||||
gathered_rows[0] * gathered_rows[1]
|
||||
), "gathered attn_mask shape does not match q/k/v"
|
||||
if gathered_mask_meta["indices"].shape[0] == 0:
|
||||
gathered_mask_meta = None
|
||||
|
||||
def masked_attention(q, k, v):
|
||||
"""Attention over the gathered sequence; heads are independent."""
|
||||
bs, seq = q.shape[0], q.shape[1]
|
||||
if tail_fa:
|
||||
assert 0 <= meta_pad_start < meta_pad_end <= seq
|
||||
cu_tail = attn_mask_meta.get("cu_seqlens_tail")
|
||||
if cu_tail is not None and meta_pad_end == seq:
|
||||
# Zero-copy tail path: run varlen FA straight over the
|
||||
# padded layout, each row split into [valid | pad]
|
||||
# segments (contiguous reshapes only, no repacking).
|
||||
assert cu_tail.numel() == 2 * bs + 1, (
|
||||
"cu_seqlens_tail does not match the batch size"
|
||||
)
|
||||
out = flash_attn_varlen_func(
|
||||
q=q.reshape(bs * seq, *q.shape[2:]),
|
||||
k=k.reshape(bs * seq, *k.shape[2:]),
|
||||
v=v.reshape(bs * seq, *v.shape[2:]),
|
||||
cu_seqlens_q=cu_tail,
|
||||
cu_seqlens_k=cu_tail,
|
||||
max_seqlen_q=attn_mask_meta["max_seqlen_tail"],
|
||||
max_seqlen_k=attn_mask_meta["max_seqlen_tail"],
|
||||
softmax_scale=self.softmax_scale,
|
||||
causal=False,
|
||||
ver=_fa_backend.fa_ver,
|
||||
).reshape(bs, seq, *q.shape[2:])
|
||||
# Match the packed paths: masked query rows read as zeros.
|
||||
out[:, meta_pad_start:].zero_()
|
||||
return out
|
||||
valid_seq = seq - (meta_pad_end - meta_pad_start)
|
||||
q_dense = torch.cat(
|
||||
[q[:, :meta_pad_start], q[:, meta_pad_end:]], dim=1
|
||||
)
|
||||
k_dense = torch.cat(
|
||||
[k[:, :meta_pad_start], k[:, meta_pad_end:]], dim=1
|
||||
)
|
||||
v_dense = torch.cat(
|
||||
[v[:, :meta_pad_start], v[:, meta_pad_end:]], dim=1
|
||||
)
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(bs + 1) * valid_seq,
|
||||
valid_seq,
|
||||
dtype=torch.int32,
|
||||
device=q.device,
|
||||
)
|
||||
out_dense = flash_attn_varlen_func(
|
||||
q=q_dense.reshape(bs * valid_seq, *q.shape[2:]),
|
||||
k=k_dense.reshape(bs * valid_seq, *k.shape[2:]),
|
||||
v=v_dense.reshape(bs * valid_seq, *v.shape[2:]),
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=valid_seq,
|
||||
max_seqlen_k=valid_seq,
|
||||
softmax_scale=self.softmax_scale,
|
||||
causal=False,
|
||||
ver=_fa_backend.fa_ver,
|
||||
).reshape(bs, seq, *q.shape[2:])
|
||||
# Match the packed paths: masked query rows read as zeros.
|
||||
out[:, meta_pad_start:].zero_()
|
||||
if sp_size > 1:
|
||||
out = _usp_output_all_to_all(out, head_dim=2)
|
||||
return out
|
||||
valid_seq = seq - (meta_pad_end - meta_pad_start)
|
||||
q_dense = torch.cat([q[:, :meta_pad_start], q[:, meta_pad_end:]], dim=1)
|
||||
k_dense = torch.cat([k[:, :meta_pad_start], k[:, meta_pad_end:]], dim=1)
|
||||
v_dense = torch.cat([v[:, :meta_pad_start], v[:, meta_pad_end:]], dim=1)
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(bs + 1) * valid_seq,
|
||||
valid_seq,
|
||||
dtype=torch.int32,
|
||||
device=q.device,
|
||||
)
|
||||
out_dense = flash_attn_varlen_func(
|
||||
q=q_dense.reshape(bs * valid_seq, *q.shape[2:]),
|
||||
k=k_dense.reshape(bs * valid_seq, *k.shape[2:]),
|
||||
v=v_dense.reshape(bs * valid_seq, *v.shape[2:]),
|
||||
cu_seqlens_q=cu_seqlens,
|
||||
cu_seqlens_k=cu_seqlens,
|
||||
max_seqlen_q=valid_seq,
|
||||
max_seqlen_k=valid_seq,
|
||||
softmax_scale=self.softmax_scale,
|
||||
causal=False,
|
||||
ver=_fa_backend.fa_ver,
|
||||
).reshape(bs, valid_seq, *q.shape[2:])
|
||||
gap_out = out_dense.new_zeros(
|
||||
bs,
|
||||
meta_pad_end - meta_pad_start,
|
||||
out_dense.shape[2],
|
||||
out_dense.shape[3],
|
||||
)
|
||||
out = torch.cat(
|
||||
[
|
||||
out_dense[:, :meta_pad_start],
|
||||
gap_out,
|
||||
out_dense[:, meta_pad_start:],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
if sp_size > 1:
|
||||
out = _usp_output_all_to_all(out, head_dim=2)
|
||||
return out
|
||||
).reshape(bs, valid_seq, *q.shape[2:])
|
||||
gap_out = out_dense.new_zeros(
|
||||
bs,
|
||||
meta_pad_end - meta_pad_start,
|
||||
out_dense.shape[2],
|
||||
out_dense.shape[3],
|
||||
)
|
||||
return torch.cat(
|
||||
[
|
||||
out_dense[:, :meta_pad_start],
|
||||
gap_out,
|
||||
out_dense[:, meta_pad_start:],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
# If NCCL timeout/deadlock occurs here, check whether
|
||||
# attn_mask is inconsistent across SP ranks (None on some, Tensor on
|
||||
# others), which causes all_gather participant mismatch. Upstream
|
||||
# mask builders must ensure all ranks produce the same mask type.
|
||||
if attn_mask is None:
|
||||
# Meta-only tail-pad caller on a non-FA fallback: the gathered
|
||||
# mask is fully determined by the pad span, no collective needed.
|
||||
gathered_mask = torch.ones(
|
||||
q.shape[0], q.shape[1], dtype=torch.bool, device=q.device
|
||||
)
|
||||
gathered_mask[:, meta_pad_start:meta_pad_end] = False
|
||||
else:
|
||||
gathered_mask = sequence_model_parallel_all_gather(
|
||||
attn_mask.contiguous(), dim=1
|
||||
)
|
||||
if (
|
||||
_VARLEN_FA_ENABLED
|
||||
and self.backend == AttentionBackendEnum.FA
|
||||
and gathered_mask.dtype
|
||||
in (torch.bool, torch.uint8, torch.int32, torch.int64)
|
||||
and q.device.type == "cuda"
|
||||
and gathered_mask.device == q.device
|
||||
and q.dtype in (torch.float16, torch.bfloat16)
|
||||
and q.shape[:2] == gathered_mask.shape == k.shape[:2] == v.shape[:2]
|
||||
):
|
||||
bs, seq = q.shape[0], q.shape[1]
|
||||
gathered_mask_meta = build_varlen_mask_meta(gathered_mask)
|
||||
indices = gathered_mask_meta["indices"]
|
||||
inv_indices = gathered_mask_meta["inv_indices"]
|
||||
assert inv_indices.shape[0] == bs * seq, (
|
||||
"gathered attn_mask shape does not match q/k/v"
|
||||
)
|
||||
if indices.shape[0] > 0:
|
||||
q_unpad, k_unpad, v_unpad = fused_pack_qkv(q, k, v, indices)
|
||||
if gathered_mask_meta is not None:
|
||||
q_unpad, k_unpad, v_unpad = fused_pack_qkv(
|
||||
q, k, v, gathered_mask_meta["indices"]
|
||||
)
|
||||
out_unpad = flash_attn_varlen_func(
|
||||
q=q_unpad,
|
||||
k=k_unpad,
|
||||
@@ -1389,33 +1407,57 @@ class USPAttention(nn.Module):
|
||||
causal=False,
|
||||
ver=_fa_backend.fa_ver,
|
||||
)
|
||||
out = fused_scatter_to_padded(out_unpad, inv_indices, bs, seq)
|
||||
if sp_size > 1:
|
||||
out = _usp_output_all_to_all(out, head_dim=2)
|
||||
return out
|
||||
return fused_scatter_to_padded(
|
||||
out_unpad, gathered_mask_meta["inv_indices"], bs, seq
|
||||
)
|
||||
|
||||
q_ = q.transpose(1, 2)
|
||||
k_ = k.transpose(1, 2)
|
||||
v_ = v.transpose(1, 2)
|
||||
mask = _prepare_sdpa_mask(gathered_mask, dtype=q_.dtype, device=q_.device)
|
||||
sdpa_context = (
|
||||
sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
|
||||
if self.allow_cudnn_sdp and q_.device.type == "cuda"
|
||||
else nullcontext()
|
||||
)
|
||||
with sdpa_context:
|
||||
out = torch.nn.functional.scaled_dot_product_attention(
|
||||
q_,
|
||||
k_,
|
||||
v_,
|
||||
attn_mask=mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
scale=self.softmax_scale,
|
||||
).transpose(1, 2)
|
||||
if sp_size > 1:
|
||||
out = _usp_output_all_to_all(out, head_dim=2)
|
||||
return out
|
||||
q_ = q.transpose(1, 2)
|
||||
k_ = k.transpose(1, 2)
|
||||
v_ = v.transpose(1, 2)
|
||||
mask = _prepare_sdpa_mask(
|
||||
gathered_mask, dtype=q_.dtype, device=q_.device
|
||||
)
|
||||
sdpa_context = (
|
||||
sdpa_kernel(_PYTORCH_DEFAULT_CUDA_SDP_BACKENDS, set_priority=True)
|
||||
if self.allow_cudnn_sdp and q_.device.type == "cuda"
|
||||
else nullcontext()
|
||||
)
|
||||
with sdpa_context:
|
||||
return torch.nn.functional.scaled_dot_product_attention(
|
||||
q_,
|
||||
k_,
|
||||
v_,
|
||||
attn_mask=mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
scale=self.softmax_scale,
|
||||
).transpose(1, 2)
|
||||
|
||||
def sequential():
|
||||
q_all, k_all, v_all = q, k, v
|
||||
if exchange:
|
||||
qkv_fast = _ipc_input_a2a_qkv(q, k, v)
|
||||
if qkv_fast is not None:
|
||||
q_all, k_all, v_all = qkv_fast
|
||||
else:
|
||||
q_all, k_all, v_all = _usp_input_all_to_all_qkv(q, k, v)
|
||||
out = masked_attention(q_all, k_all, v_all)
|
||||
if sp_size > 1:
|
||||
out = _usp_output_all_to_all(out, head_dim=2)
|
||||
return out
|
||||
|
||||
if exchange:
|
||||
out = self._pipelined_ulysses(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
masked_attention,
|
||||
sequential,
|
||||
layout="masked-tail" if tail_fa else "masked",
|
||||
)
|
||||
if out is not None:
|
||||
return out
|
||||
return sequential()
|
||||
|
||||
if effective_skip_sp or get_sequence_parallel_world_size() == 1:
|
||||
# No sequence parallelism, just run local attention.
|
||||
@@ -1444,6 +1486,27 @@ class USPAttention(nn.Module):
|
||||
q, k, v, ctx_attn_metadata, num_replicated_kv_prefix
|
||||
)
|
||||
|
||||
if (
|
||||
sp_size > 1
|
||||
and not qkv_pre_all_to_all
|
||||
and get_ring_parallel_world_size() == 1
|
||||
and not self.enable_packed_qkv_input_a2a
|
||||
):
|
||||
|
||||
def attend(q_group, k_group, v_group):
|
||||
return self.attn_impl.forward(
|
||||
q_group, k_group, v_group, ctx_attn_metadata
|
||||
)
|
||||
|
||||
def sequential():
|
||||
qs, ks, vs = _usp_input_all_to_all_qkv(q, k, v)
|
||||
out = self.attn_impl.forward(qs, ks, vs, ctx_attn_metadata)
|
||||
return _usp_output_all_to_all(out, head_dim=2)
|
||||
|
||||
out = self._pipelined_ulysses(q, k, v, attend, sequential, layout="dense")
|
||||
if out is not None:
|
||||
return out
|
||||
|
||||
# Ulysses-style All-to-All for sequence/head sharding
|
||||
if sp_size > 1 and not qkv_pre_all_to_all:
|
||||
# -> [B, S, H_local, D]
|
||||
@@ -1482,6 +1545,91 @@ class USPAttention(nn.Module):
|
||||
|
||||
return out
|
||||
|
||||
def _pipelined_ulysses(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
attend,
|
||||
sequential,
|
||||
*,
|
||||
layout: str,
|
||||
replicated=(None, None, None),
|
||||
replicated_first: bool = True,
|
||||
) -> torch.Tensor | None:
|
||||
"""Ulysses attention with the exchange pipelined over head groups.
|
||||
|
||||
The building block every Ulysses path of this layer goes through: q/k/v
|
||||
are the call's ``[B, S_local, H, D]`` sequence shards and ``replicated``
|
||||
the rows every rank holds (see ``ulysses_pipelined_attention``).
|
||||
``attend(q, k, v)`` is the path's attention over the gathered sequence,
|
||||
given one head group at a time, and ``sequential()`` the path's own
|
||||
exchange-attend-exchange: the first call of each shape runs both and
|
||||
keeps the pipeline only if every rank sees the same bytes. Returns
|
||||
None, so the caller runs ``sequential()`` itself, when this call
|
||||
cannot pipeline.
|
||||
"""
|
||||
groups = envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS
|
||||
if (
|
||||
not q.is_cuda
|
||||
or groups in (0, 1)
|
||||
or self.backend not in PIPELINED_ATTENTION_BACKENDS
|
||||
or torch.compiler.is_compiling()
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
return None
|
||||
# shapes the transport already turned down stay down (too short to
|
||||
# split, buffers that do not fit, a backend that failed the first-sight
|
||||
# check); skip straight to the sequential path for them
|
||||
key = (
|
||||
layout,
|
||||
groups,
|
||||
self.backend,
|
||||
q.shape,
|
||||
k.shape,
|
||||
v.shape,
|
||||
q.stride(),
|
||||
k.stride(),
|
||||
v.stride(),
|
||||
tuple(None if t is None else (t.shape, t.stride()) for t in replicated),
|
||||
replicated_first,
|
||||
)
|
||||
if key in self._pipeline_declined:
|
||||
return None
|
||||
if (
|
||||
get_ring_parallel_world_size() > 1
|
||||
or get_request_skip_softmax_params() is not None
|
||||
):
|
||||
return None
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
ulysses_pipelined_attention,
|
||||
)
|
||||
|
||||
def reference():
|
||||
# once per shape: leave no staging behind that later calls never use
|
||||
with uncached_a2a_staging():
|
||||
return sequential()
|
||||
|
||||
out = ulysses_pipelined_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attend,
|
||||
groups,
|
||||
sequential=reference,
|
||||
signature=(
|
||||
type(self.attn_impl).__name__,
|
||||
self.causal,
|
||||
self.softmax_scale,
|
||||
layout,
|
||||
),
|
||||
replicated=replicated,
|
||||
replicated_first=replicated_first,
|
||||
)
|
||||
if out is None:
|
||||
self._pipeline_declined.add(key)
|
||||
return out
|
||||
|
||||
def _forward_ring_tail_pad(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -1740,8 +1888,6 @@ class USPAttention(nn.Module):
|
||||
k_rep, k_shard = k[:, :num_rep], k[:, num_rep:]
|
||||
v_rep, v_shard = v[:, :num_rep], v[:, num_rep:]
|
||||
|
||||
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
|
||||
|
||||
joint_mask = joint_mask_meta = None
|
||||
if attn_mask is not None:
|
||||
cache_key = (
|
||||
@@ -1769,44 +1915,68 @@ class USPAttention(nn.Module):
|
||||
joint_mask_meta,
|
||||
)
|
||||
|
||||
# Q and KV can have different head counts (GQA), so slice each replicated
|
||||
# prefix by its own per-rank head shard to match the all-to-all'd suffix.
|
||||
# For MHA (kv heads == q heads) this is identical to the q shard.
|
||||
h_local = q_shard.shape[2]
|
||||
kv_h_local = k_shard.shape[2]
|
||||
h_start = u_rank * h_local
|
||||
kv_h_start = u_rank * kv_h_local
|
||||
q_rep = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
|
||||
k_rep = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
v_rep = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
def attend(q_group, k_group, v_group):
|
||||
if joint_mask is not None:
|
||||
return self._masked_local_attention(
|
||||
q_group,
|
||||
k_group,
|
||||
v_group,
|
||||
joint_mask,
|
||||
attn_mask_meta=joint_mask_meta,
|
||||
)
|
||||
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
|
||||
|
||||
q = torch.cat([q_rep, q_shard], dim=1)
|
||||
out = self._replicated_kv_attention(
|
||||
q,
|
||||
def sequential():
|
||||
q_gathered, k_gathered, v_gathered = _usp_input_all_to_all_qkv(
|
||||
q_shard, k_shard, v_shard
|
||||
)
|
||||
# Q and KV can have different head counts (GQA), so slice each
|
||||
# replicated prefix by its own per-rank head shard to match the
|
||||
# all-to-all'd suffix. For MHA (kv heads == q heads) this is
|
||||
# identical to the q shard.
|
||||
h_local = q_gathered.shape[2]
|
||||
kv_h_local = k_gathered.shape[2]
|
||||
h_start = u_rank * h_local
|
||||
kv_h_start = u_rank * kv_h_local
|
||||
q_local = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
|
||||
k_local = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
v_local = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
|
||||
out = self._replicated_kv_attention(
|
||||
torch.cat([q_local, q_gathered], dim=1),
|
||||
k_gathered,
|
||||
v_gathered,
|
||||
k_local,
|
||||
v_local,
|
||||
ctx_attn_metadata,
|
||||
attn_mask=joint_mask,
|
||||
attn_mask_meta=joint_mask_meta,
|
||||
)
|
||||
|
||||
out_rep = out[:, :num_rep]
|
||||
out_shard = _usp_output_all_to_all(out[:, num_rep:], head_dim=2)
|
||||
|
||||
if sp_size > 1:
|
||||
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
|
||||
torch.distributed.all_gather(
|
||||
gathered,
|
||||
out_rep.contiguous(),
|
||||
group=get_sp_group().ulysses_group,
|
||||
)
|
||||
out_rep = torch.cat(gathered, dim=2)
|
||||
|
||||
return torch.cat([out_rep, out_shard], dim=1)
|
||||
|
||||
out = self._pipelined_ulysses(
|
||||
q_shard,
|
||||
k_shard,
|
||||
v_shard,
|
||||
k_rep,
|
||||
v_rep,
|
||||
ctx_attn_metadata,
|
||||
attn_mask=joint_mask,
|
||||
attn_mask_meta=joint_mask_meta,
|
||||
attend,
|
||||
sequential,
|
||||
layout="replicated-prefix" + ("-masked" if joint_mask is not None else ""),
|
||||
replicated=(q_rep, k_rep, v_rep),
|
||||
)
|
||||
|
||||
out_rep = out[:, :num_rep]
|
||||
out_shard = out[:, num_rep:]
|
||||
|
||||
out_shard = _usp_output_all_to_all(out_shard, head_dim=2)
|
||||
|
||||
if sp_size > 1:
|
||||
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
|
||||
torch.distributed.all_gather(
|
||||
gathered,
|
||||
out_rep.contiguous(),
|
||||
group=get_sp_group().ulysses_group,
|
||||
)
|
||||
out_rep = torch.cat(gathered, dim=2)
|
||||
|
||||
return torch.cat([out_rep, out_shard], dim=1)
|
||||
return out if out is not None else sequential()
|
||||
|
||||
def _replicated_kv_attention(
|
||||
self,
|
||||
@@ -1998,32 +2168,52 @@ class USPAttention(nn.Module):
|
||||
"""split form avoids materializing full K/V before Ulysses all-to-all"""
|
||||
u_rank = get_ulysses_parallel_rank()
|
||||
|
||||
if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1:
|
||||
q, k_shard, v_shard = async_a2a_communicate(
|
||||
[q, k_shard, v_shard],
|
||||
get_ulysses_parallel_world_size(),
|
||||
get_sp_group().ulysses_group,
|
||||
self._get_usp_a2a_stream(),
|
||||
local_seq_2_local_head=True,
|
||||
def attend(q_group, k_group, v_group):
|
||||
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
|
||||
|
||||
def sequential():
|
||||
if q.device.type == "cuda" and get_ulysses_parallel_world_size() > 1:
|
||||
q_gathered, k_gathered, v_gathered = async_a2a_communicate(
|
||||
[q, k_shard, v_shard],
|
||||
get_ulysses_parallel_world_size(),
|
||||
get_sp_group().ulysses_group,
|
||||
self._get_usp_a2a_stream(),
|
||||
local_seq_2_local_head=True,
|
||||
)
|
||||
q_gathered = q_gathered.contiguous()
|
||||
k_gathered = k_gathered.contiguous()
|
||||
v_gathered = v_gathered.contiguous()
|
||||
else:
|
||||
q_gathered = _usp_input_all_to_all(q, head_dim=2)
|
||||
k_gathered = _usp_input_all_to_all(k_shard, head_dim=2)
|
||||
v_gathered = _usp_input_all_to_all(v_shard, head_dim=2)
|
||||
|
||||
h_kv_local = k_gathered.shape[2]
|
||||
h_start = u_rank * h_kv_local
|
||||
h_end = h_start + h_kv_local
|
||||
k_local = k_rep[:, :, h_start:h_end, :].contiguous()
|
||||
v_local = v_rep[:, :, h_start:h_end, :].contiguous()
|
||||
|
||||
out = self._replicated_kv_attention(
|
||||
q_gathered,
|
||||
k_gathered,
|
||||
v_gathered,
|
||||
k_local,
|
||||
v_local,
|
||||
ctx_attn_metadata,
|
||||
)
|
||||
q = q.contiguous()
|
||||
k_shard = k_shard.contiguous()
|
||||
v_shard = v_shard.contiguous()
|
||||
else:
|
||||
q = _usp_input_all_to_all(q, head_dim=2)
|
||||
k_shard = _usp_input_all_to_all(k_shard, head_dim=2)
|
||||
v_shard = _usp_input_all_to_all(v_shard, head_dim=2)
|
||||
return _usp_output_all_to_all(out, head_dim=2)
|
||||
|
||||
h_kv_local = k_shard.shape[2]
|
||||
h_start = u_rank * h_kv_local
|
||||
h_end = h_start + h_kv_local
|
||||
k_rep = k_rep[:, :, h_start:h_end, :].contiguous()
|
||||
v_rep = v_rep[:, :, h_start:h_end, :].contiguous()
|
||||
|
||||
out = self._replicated_kv_attention(
|
||||
q, k_shard, v_shard, k_rep, v_rep, ctx_attn_metadata
|
||||
out = self._pipelined_ulysses(
|
||||
q,
|
||||
k_shard,
|
||||
v_shard,
|
||||
attend,
|
||||
sequential,
|
||||
layout="replicated-kv-prefix",
|
||||
replicated=(None, k_rep, v_rep),
|
||||
)
|
||||
return _usp_output_all_to_all(out, head_dim=2)
|
||||
return out if out is not None else sequential()
|
||||
|
||||
def _forward_with_replicated_suffix(
|
||||
self,
|
||||
@@ -2049,37 +2239,58 @@ class USPAttention(nn.Module):
|
||||
k_shard, k_rep = k[:, :-num_rep], k[:, -num_rep:]
|
||||
v_shard, v_rep = v[:, :-num_rep], v[:, -num_rep:]
|
||||
|
||||
q_shard, k_shard, v_shard = _usp_input_all_to_all_qkv(q_shard, k_shard, v_shard)
|
||||
def attend(q_group, k_group, v_group):
|
||||
return self.attn_impl.forward(q_group, k_group, v_group, ctx_attn_metadata)
|
||||
|
||||
h_local = q_shard.shape[2]
|
||||
kv_h_local = k_shard.shape[2]
|
||||
h_start = u_rank * h_local
|
||||
kv_h_start = u_rank * kv_h_local
|
||||
q_rep = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
|
||||
k_rep = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
v_rep = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
|
||||
q = torch.cat([q_shard, q_rep], dim=1)
|
||||
out = self._replicated_kv_attention(
|
||||
q, k_shard, v_shard, k_rep, v_rep, ctx_attn_metadata, rep_first=False
|
||||
)
|
||||
|
||||
out_shard = out[:, :-num_rep]
|
||||
out_rep = out[:, -num_rep:]
|
||||
|
||||
out_shard = _usp_output_all_to_all(out_shard, head_dim=2)
|
||||
|
||||
sp_size = get_ulysses_parallel_world_size()
|
||||
if sp_size > 1:
|
||||
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
|
||||
torch.distributed.all_gather(
|
||||
gathered,
|
||||
out_rep.contiguous(),
|
||||
group=get_sp_group().ulysses_group,
|
||||
def sequential():
|
||||
q_gathered, k_gathered, v_gathered = _usp_input_all_to_all_qkv(
|
||||
q_shard, k_shard, v_shard
|
||||
)
|
||||
out_rep = torch.cat(gathered, dim=2)
|
||||
|
||||
return torch.cat([out_shard, out_rep], dim=1)
|
||||
h_local = q_gathered.shape[2]
|
||||
kv_h_local = k_gathered.shape[2]
|
||||
h_start = u_rank * h_local
|
||||
kv_h_start = u_rank * kv_h_local
|
||||
q_local = q_rep[:, :, h_start : h_start + h_local, :].contiguous()
|
||||
k_local = k_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
v_local = v_rep[:, :, kv_h_start : kv_h_start + kv_h_local, :].contiguous()
|
||||
|
||||
out = self._replicated_kv_attention(
|
||||
torch.cat([q_gathered, q_local], dim=1),
|
||||
k_gathered,
|
||||
v_gathered,
|
||||
k_local,
|
||||
v_local,
|
||||
ctx_attn_metadata,
|
||||
rep_first=False,
|
||||
)
|
||||
|
||||
out_shard = _usp_output_all_to_all(out[:, :-num_rep], head_dim=2)
|
||||
out_rep = out[:, -num_rep:]
|
||||
|
||||
sp_size = get_ulysses_parallel_world_size()
|
||||
if sp_size > 1:
|
||||
gathered = [torch.empty_like(out_rep) for _ in range(sp_size)]
|
||||
torch.distributed.all_gather(
|
||||
gathered,
|
||||
out_rep.contiguous(),
|
||||
group=get_sp_group().ulysses_group,
|
||||
)
|
||||
out_rep = torch.cat(gathered, dim=2)
|
||||
|
||||
return torch.cat([out_shard, out_rep], dim=1)
|
||||
|
||||
out = self._pipelined_ulysses(
|
||||
q_shard,
|
||||
k_shard,
|
||||
v_shard,
|
||||
attend,
|
||||
sequential,
|
||||
layout="replicated-suffix",
|
||||
replicated=(q_rep, k_rep, v_rep),
|
||||
replicated_first=False,
|
||||
)
|
||||
return out if out is not None else sequential()
|
||||
|
||||
|
||||
class _BCGBoxedTupleOutput:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -28,6 +29,25 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_A2A_STAGING_BUFFERS: dict[tuple[str, torch.dtype, int], torch.Tensor] = {}
|
||||
# depth of open uncached_a2a_staging() blocks
|
||||
_UNCACHED_STAGING = 0
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def uncached_a2a_staging():
|
||||
"""Run the Ulysses exchanges inside the block over NCCL on per-call buffers.
|
||||
|
||||
For a one-off exchange the steady state never repeats (the pipelined
|
||||
attention's first-sight reference): the staging cache and the IPC
|
||||
transports' staging pairs would otherwise stay allocated at its size. An
|
||||
all-to-all is a permutation, so the transport never changes its bytes.
|
||||
"""
|
||||
global _UNCACHED_STAGING
|
||||
_UNCACHED_STAGING += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_UNCACHED_STAGING -= 1
|
||||
|
||||
|
||||
def drop_a2a_staging_buffers() -> None:
|
||||
@@ -59,7 +79,8 @@ def _a2a_staging_buffer(
|
||||
not be shared with eager replays.
|
||||
"""
|
||||
if (
|
||||
torch.is_grad_enabled()
|
||||
_UNCACHED_STAGING
|
||||
or torch.is_grad_enabled()
|
||||
or torch.compiler.is_compiling()
|
||||
or device.type != "cuda"
|
||||
or torch.cuda.is_current_stream_capturing()
|
||||
@@ -88,7 +109,7 @@ def _usp_all_to_all_single(x: torch.Tensor, role: str | None = None) -> torch.Te
|
||||
assert ulysses_pg is not None, "Ulysses process group is not initialized."
|
||||
x_shape = x.shape
|
||||
x = x.flatten().contiguous()
|
||||
if x.is_cuda:
|
||||
if x.is_cuda and not _UNCACHED_STAGING:
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
ipc_a2a_multi_ready,
|
||||
@@ -148,6 +169,8 @@ def _ipc_ready_group():
|
||||
ipc_a2a_ready,
|
||||
)
|
||||
|
||||
if _UNCACHED_STAGING:
|
||||
return None
|
||||
group = get_sp_group().ulysses_group
|
||||
return group if ipc_a2a_ready(group) else None
|
||||
|
||||
|
||||
@@ -1562,6 +1562,7 @@ STANDALONE_FILES = {
|
||||
"../single_test_file/test_ar_models.py",
|
||||
"../single_test_file/test_ipc_a2a_2_gpu.py",
|
||||
"../single_test_file/test_ipc_a2a_multi_gpu.py",
|
||||
"../single_test_file/test_usp_pipelined_attention_multi_gpu.py",
|
||||
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py",
|
||||
"../single_test_file/test_encoder_fold_srt_2_gpu.py",
|
||||
"../single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py",
|
||||
@@ -1605,6 +1606,8 @@ STANDALONE_FILE_EST_TIMES = {
|
||||
"../single_test_file/test_ipc_a2a_2_gpu.py": 240.0,
|
||||
# 2 ranks here (4 ranks skip below four GPUs); ~50 s on H200 with warm JIT
|
||||
"../single_test_file/test_ipc_a2a_multi_gpu.py": 180.0,
|
||||
# 2 ranks here (4 ranks skip below four GPUs); ~45 s on B300 with warm JIT
|
||||
"../single_test_file/test_usp_pipelined_attention_multi_gpu.py": 120.0,
|
||||
"../single_test_file/test_encoder_fold_srt_linear_2_gpu.py": 120.0,
|
||||
"../single_test_file/test_encoder_fold_srt_2_gpu.py": 240.0,
|
||||
# ~60 s locally with a warm HF cache (load + one capture + 4 steps);
|
||||
|
||||
@@ -27,6 +27,24 @@ def _attend(q, k, v):
|
||||
return (q.float() * 2 + k.flip(0).float() - v.roll(1, 0).float()).to(q.dtype)
|
||||
|
||||
|
||||
def _attend_rows(q, k, v):
|
||||
"""`_attend` for [batch, rows, heads, head_dim] parts whose k/v may carry more
|
||||
rows (a replicated KV prefix) and fewer heads (GQA) than q."""
|
||||
rep = q.shape[2] // k.shape[2]
|
||||
k = k.repeat_interleave(rep, dim=2).float()
|
||||
v = v.repeat_interleave(rep, dim=2).float()
|
||||
rows = q.shape[1]
|
||||
# elementwise only (a reduction could round differently per head count):
|
||||
# the leading and the trailing rows both read, so every row counts
|
||||
return (
|
||||
q.float() * 2
|
||||
+ k[:, :rows]
|
||||
+ k.flip(1)[:, :rows]
|
||||
- v.roll(1, 1)[:, -rows:]
|
||||
+ v[:, :rows] * 0.5
|
||||
).to(q.dtype)
|
||||
|
||||
|
||||
def _worker() -> int:
|
||||
import torch.distributed as dist
|
||||
|
||||
@@ -69,7 +87,7 @@ def _worker() -> int:
|
||||
if world == 2:
|
||||
cases.append((128, 16, 64, 8))
|
||||
for s_local, heads, head_dim, groups in cases:
|
||||
# three calls in a row: the slots alternate and the counters advance
|
||||
# three calls in a row: the slots are reused and the counters advance
|
||||
for call in range(3):
|
||||
torch.manual_seed(
|
||||
1000 * call + s_local
|
||||
@@ -132,23 +150,146 @@ def _worker() -> int:
|
||||
f"filled pipeline {(s_local, heads, head_dim, groups)} call {call}"
|
||||
)
|
||||
|
||||
# groups=-1 picks a count that divides the heads per rank (4 for 16 heads at
|
||||
# 2 ranks, 2 at 4) and still matches
|
||||
# groups=-1 picks the most groups that divide the heads per rank and whose
|
||||
# calls still fill four waves of 128-row tiles, and still matches; a shape too
|
||||
# small to fill them keeps the sequential exchange
|
||||
sms = torch.cuda.get_device_properties(rank).multi_processor_count
|
||||
s_auto = 4 * sms * 128 // world # one head per group fills four waves
|
||||
torch.manual_seed(77)
|
||||
full = [
|
||||
torch.randn(64 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
|
||||
torch.randn(s_auto * world, 16, 64, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(3)
|
||||
]
|
||||
q, k, v = (t.narrow(0, rank * 64, 64).contiguous() for t in full)
|
||||
q, k, v = (t.narrow(0, rank * s_auto, s_auto).contiguous() for t in full)
|
||||
got = ulysses_pipelined_attention(q, k, v, _attend, -1)
|
||||
if got is None or not torch.equal(sequential(q, k, v), got):
|
||||
failures.append("auto group count")
|
||||
small = torch.randn(64, 16, 64, dtype=torch.bfloat16, device="cuda")
|
||||
if ulysses_pipelined_attention(small, small, small, _attend, -1) is not None:
|
||||
failures.append("auto pipelined a shape too small to fill the GPU")
|
||||
|
||||
# sequential=: the first call per shape and signature also runs the
|
||||
# sequential exchange and keeps the pipeline only on a byte-for-byte match
|
||||
torch.manual_seed(78)
|
||||
full = [
|
||||
torch.randn(80 * world, 16, 64, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(3)
|
||||
]
|
||||
q, k, v = (t.narrow(0, rank * 80, 80).contiguous() for t in full)
|
||||
ref = sequential(q, k, v)
|
||||
for call in range(2):
|
||||
got = ulysses_pipelined_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
_attend,
|
||||
2,
|
||||
sequential=lambda: sequential(q, k, v),
|
||||
signature="same",
|
||||
)
|
||||
if got is None or not torch.equal(ref, got):
|
||||
failures.append(f"verified pipeline call {call}")
|
||||
off = ref + 1 # a "sequential" result the pipeline cannot reproduce
|
||||
got = ulysses_pipelined_attention(
|
||||
q, k, v, _attend, 2, sequential=lambda: off, signature="other"
|
||||
)
|
||||
if got is None or not torch.equal(got, off):
|
||||
failures.append("a mismatching first call did not return the sequential result")
|
||||
if (
|
||||
ulysses_pipelined_attention(
|
||||
q, k, v, _attend, 2, sequential=lambda: off, signature="other"
|
||||
)
|
||||
is not None
|
||||
):
|
||||
failures.append("a mismatched shape kept pipelining")
|
||||
|
||||
# a head count the groups cannot split must decline, not mis-shard
|
||||
q = torch.randn(32, 12, 64, dtype=torch.bfloat16, device="cuda")
|
||||
if ulysses_pipelined_attention(q, q, q, _attend, 4) is not None:
|
||||
failures.append("pipelined accepted 12 heads in 4 groups")
|
||||
|
||||
# batch, replicated rows (prefix, suffix, KV-only prefix) and GQA: the output
|
||||
# rows match attention over the global layout, computed with every head
|
||||
def layout_case(batch, s_local, heads, kv_heads, rep, rep_kv, first, groups, seed):
|
||||
torch.manual_seed(seed)
|
||||
dims = 32
|
||||
shard = lambda h: torch.randn(
|
||||
batch, s_local * world, h, dims, device="cuda"
|
||||
).bfloat16()
|
||||
full = [shard(heads), shard(kv_heads), shard(kv_heads)]
|
||||
reps = [
|
||||
torch.randn(batch, n, h, dims, device="cuda").bfloat16() if n else None
|
||||
for n, h in ((rep, heads), (rep_kv, kv_heads), (rep_kv, kv_heads))
|
||||
]
|
||||
mine = [t[:, rank * s_local : (rank + 1) * s_local].contiguous() for t in full]
|
||||
|
||||
def joined(t, r):
|
||||
if r is None:
|
||||
return t
|
||||
return torch.cat([r, t] if first else [t, r], dim=1)
|
||||
|
||||
out_all = _attend_rows(*(joined(t, r) for t, r in zip(full, reps)))
|
||||
lo = (rep if first else 0) + rank * s_local
|
||||
own = out_all[:, lo : lo + s_local]
|
||||
if rep:
|
||||
rep_rows = out_all[:, :rep] if first else out_all[:, -rep:]
|
||||
own = torch.cat([rep_rows, own] if first else [own, rep_rows], dim=1)
|
||||
got = ulysses_pipelined_attention(
|
||||
*mine,
|
||||
_attend_rows,
|
||||
groups,
|
||||
replicated=tuple(reps),
|
||||
replicated_first=first,
|
||||
)
|
||||
label = f"layout b{batch} h{heads}/{kv_heads} rep {rep}/{rep_kv} first={first} g{groups}"
|
||||
if got is None:
|
||||
failures.append(f"{label} returned None")
|
||||
elif not torch.equal(got, own):
|
||||
failures.append(label)
|
||||
|
||||
for call in range(2): # every layout reuses the one pool
|
||||
layout_case(2, 48, 16, 16, 0, 0, True, 2, 300 + call)
|
||||
layout_case(1, 40, 16, 16, 9, 9, True, 2, 310 + call)
|
||||
layout_case(2, 40, 16, 16, 7, 7, False, 2, 320 + call)
|
||||
layout_case(1, 40, 16, 16, 0, 11, True, 2, 330 + call)
|
||||
layout_case(2, 40, 16, 8, 5, 5, True, 2, 340 + call)
|
||||
|
||||
# one pool per dtype serves every plan above, sized for the largest of them
|
||||
pools = IPC_A2A_MULTI.pools
|
||||
if list(pools) != [torch.bfloat16]:
|
||||
failures.append(f"pipeline pools per dtype: {list(pools)}")
|
||||
else:
|
||||
slots = list(IPC_A2A_MULTI.slots.values())
|
||||
widest = (
|
||||
max(s.in_total for s in slots),
|
||||
max(s.out_total for s in slots),
|
||||
max(s.plan.groups for s in slots),
|
||||
)
|
||||
if pools[torch.bfloat16].sizes != widest:
|
||||
failures.append(f"pool sizes {pools[torch.bfloat16].sizes} != {widest}")
|
||||
|
||||
# one set of slots is enough: a peer cannot write a call's rows or output
|
||||
# blocks before this rank has read the previous call's, however far ahead
|
||||
# it runs. Rank 0 attends slowly; alternating plans share the pool.
|
||||
def slow_attend(q, k, v):
|
||||
if rank == 0:
|
||||
torch.cuda._sleep(20_000_000)
|
||||
return _attend(q, k, v)
|
||||
|
||||
issued = []
|
||||
for call in range(6): # queued back to back; compared only afterwards
|
||||
s_local, heads, groups = ((96, 16, 2), (40, 32, 4))[call % 2]
|
||||
torch.manual_seed(4000 + call)
|
||||
full = [
|
||||
torch.randn(s_local * world, heads, 64, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(3)
|
||||
]
|
||||
parts = [t.narrow(0, rank * s_local, s_local).contiguous() for t in full]
|
||||
issued.append((parts, ulysses_pipelined_attention(*parts, slow_attend, groups)))
|
||||
for call, (parts, got) in enumerate(issued):
|
||||
if got is None or not torch.equal(sequential(*parts), got):
|
||||
failures.append(f"skewed back-to-back call {call}")
|
||||
|
||||
# the plain N-rank exchange behind _usp_all_to_all_single
|
||||
for numel in (world * 1024, world * 4096 * 33):
|
||||
x = torch.randn(numel, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
"""Every Ulysses path of USPAttention gives the same bytes with the exchange
|
||||
pipelined over head groups as with the sequential exchange.
|
||||
|
||||
The pipeline only engages on a same-host Ulysses group with peer-to-peer access,
|
||||
so nothing in the single-GPU suite reaches it. Runs on 2 ranks, and on 4 when
|
||||
four GPUs are visible:
|
||||
|
||||
pytest -v python/sglang/multimodal_gen/test/single_test_file/test_usp_pipelined_attention_multi_gpu.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
def _worker() -> int:
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
|
||||
rank = int(os.environ["RANK"])
|
||||
world = int(os.environ["WORLD_SIZE"])
|
||||
torch.cuda.set_device(rank)
|
||||
maybe_init_distributed_environment_and_model_parallel(
|
||||
tp_size=1, sp_size=world, ulysses_degree=world
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
|
||||
IPC_A2A,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a_multi import (
|
||||
IPC_A2A_MULTI,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers import usp
|
||||
from sglang.multimodal_gen.runtime.layers.attention.layer import (
|
||||
PIPELINED_ATTENTION_BACKENDS,
|
||||
USPAttention,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
|
||||
from sglang.multimodal_gen.test.unit.conftest import _make_unit_server_args
|
||||
|
||||
set_global_server_args(_make_unit_server_args())
|
||||
peers = [d for d in range(world) if d != rank]
|
||||
if not all(torch.cuda.can_device_access_peer(rank, d) for d in peers):
|
||||
print("SKIP no peer-to-peer access between the devices", flush=True)
|
||||
return 0
|
||||
|
||||
heads, kv_heads, dim, s_local, n_rep = 16, 8, 64, 96, 24
|
||||
failures, ran = [], []
|
||||
backends = [b for b in PIPELINED_ATTENTION_BACKENDS]
|
||||
|
||||
def shard(batch, rows, h, seed):
|
||||
torch.manual_seed(seed) # identical full tensors on every rank
|
||||
full = torch.randn(batch, rows * world, h, dim, device="cuda").bfloat16()
|
||||
return full[:, rank * rows : (rank + 1) * rows].contiguous()
|
||||
|
||||
def replicated(batch, rows, h, seed):
|
||||
torch.manual_seed(seed)
|
||||
return torch.randn(batch, rows, h, dim, device="cuda").bfloat16()
|
||||
|
||||
def both(label, call):
|
||||
"""`call()` sequentially and pipelined (two forced groups) must agree."""
|
||||
outs = []
|
||||
for groups in (0, 2):
|
||||
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = groups
|
||||
with torch.inference_mode(), set_forward_context(0, None, None):
|
||||
outs.append(call())
|
||||
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = -1
|
||||
if not torch.equal(outs[0], outs[1]):
|
||||
failures.append(label)
|
||||
ran.append(label)
|
||||
|
||||
for backend in backends:
|
||||
try:
|
||||
layer = USPAttention(
|
||||
num_heads=heads,
|
||||
head_size=dim,
|
||||
causal=False,
|
||||
required_attention_backend=backend,
|
||||
prefix=f"test.{backend.name}",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - a backend this GPU cannot build
|
||||
print(f"backend {backend.name} unavailable here: {e}", flush=True)
|
||||
continue
|
||||
if layer.backend is not backend:
|
||||
print(
|
||||
f"backend {backend.name} resolved to {layer.backend.name}", flush=True
|
||||
)
|
||||
continue
|
||||
name = backend.name
|
||||
for batch in (1, 2):
|
||||
q, k, v = (shard(batch, s_local, heads, 10 + i) for i in range(3))
|
||||
both(f"{name} dense b{batch}", lambda: layer(q, k, v))
|
||||
|
||||
rep = [replicated(batch, n_rep, heads, 20 + i) for i in range(3)]
|
||||
joined = [torch.cat([r, t], dim=1) for r, t in zip(rep, (q, k, v))]
|
||||
both(
|
||||
f"{name} replicated prefix b{batch}",
|
||||
lambda: layer(*joined, num_replicated_prefix=n_rep),
|
||||
)
|
||||
mask = torch.ones(batch, n_rep + s_local, dtype=torch.bool, device="cuda")
|
||||
mask[:, n_rep // 2 : n_rep] = False # padded text tokens
|
||||
both(
|
||||
f"{name} replicated prefix + key mask b{batch}",
|
||||
lambda: layer(*joined, num_replicated_prefix=n_rep, attn_mask=mask),
|
||||
)
|
||||
trailing = [torch.cat([t, r], dim=1) for r, t in zip(rep, (q, k, v))]
|
||||
both(
|
||||
f"{name} replicated suffix b{batch}",
|
||||
lambda: layer(*trailing, num_replicated_suffix=n_rep),
|
||||
)
|
||||
k_rep, v_rep = (replicated(batch, n_rep, heads, 30 + i) for i in range(2))
|
||||
both(
|
||||
f"{name} replicated KV prefix b{batch}",
|
||||
lambda: layer.forward_with_replicated_kv_prefix(q, k_rep, v_rep, k, v),
|
||||
)
|
||||
key_mask = torch.ones(batch, s_local, dtype=torch.bool, device="cuda")
|
||||
key_mask[:, -s_local // 4 :] = rank % 2 == 0 # ragged across ranks
|
||||
both(
|
||||
f"{name} gathered key mask b{batch}",
|
||||
lambda: layer(q, k, v, attn_mask=key_mask),
|
||||
)
|
||||
gqa = USPAttention(
|
||||
num_heads=heads,
|
||||
head_size=dim,
|
||||
num_kv_heads=kv_heads,
|
||||
causal=False,
|
||||
required_attention_backend=backend,
|
||||
prefix=f"test.gqa.{backend.name}",
|
||||
)
|
||||
q = shard(1, s_local, heads, 40)
|
||||
k, v = (shard(1, s_local, kv_heads, 41 + i) for i in range(2))
|
||||
both(f"{name} GQA dense", lambda: gqa(q, k, v))
|
||||
|
||||
# the first call of a shape checks the pipeline against a sequential
|
||||
# exchange on per-call buffers: no staging sized for it stays behind
|
||||
usp.drop_a2a_staging_buffers()
|
||||
IPC_A2A.drop_staging()
|
||||
checked = len(IPC_A2A_MULTI.verified)
|
||||
q, k, v = (shard(1, 2 * s_local, heads, 50 + i) for i in range(3))
|
||||
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = 2
|
||||
with torch.inference_mode(), set_forward_context(0, None, None):
|
||||
layer(q, k, v)
|
||||
envs.SGLANG_DIFFUSION_ULYSSES_PIPELINE_GROUPS = -1
|
||||
if len(IPC_A2A_MULTI.verified) != checked + 1:
|
||||
failures.append(f"{name}: a new shape was not checked")
|
||||
if usp._A2A_STAGING_BUFFERS or IPC_A2A.staging:
|
||||
failures.append(
|
||||
f"{name}: the first-sight check left sequential staging behind "
|
||||
f"({list(usp._A2A_STAGING_BUFFERS)}, {len(IPC_A2A.staging)} IPC pairs)"
|
||||
)
|
||||
|
||||
# a comparison where every arm quietly fell back would pass while testing
|
||||
# nothing: the pipeline must have run and verified every layout
|
||||
if not ran:
|
||||
failures.append("no pipelined backend could be built")
|
||||
if IPC_A2A_MULTI.mismatched:
|
||||
failures.append(
|
||||
f"pipeline disagreed with the sequential exchange: {IPC_A2A_MULTI.mismatched}"
|
||||
)
|
||||
if len(IPC_A2A_MULTI.verified) < len({label.split(" b")[0] for label in ran}):
|
||||
failures.append(
|
||||
f"pipeline verified {len(IPC_A2A_MULTI.verified)} layouts for {len(ran)} cases"
|
||||
)
|
||||
|
||||
verdict = torch.tensor([len(failures)], device="cuda")
|
||||
dist.all_reduce(verdict)
|
||||
if failures:
|
||||
print(f"rank{rank} MISMATCH: {failures}", flush=True)
|
||||
if rank == 0:
|
||||
print(f"ran: {ran}", flush=True)
|
||||
print(
|
||||
f"USP_PIPELINE_PARITY {'FAIL' if verdict.item() else 'PASS'} world={world}",
|
||||
flush=True,
|
||||
)
|
||||
dist.barrier()
|
||||
dist.destroy_process_group()
|
||||
return 1 if verdict.item() else 0
|
||||
|
||||
|
||||
class TestUspPipelinedAttentionMultiGpu(CustomTestCase):
|
||||
def _run(self, world: int, port: int):
|
||||
if not current_platform.is_cuda():
|
||||
self.skipTest("the copy-engine pipeline is CUDA-only")
|
||||
if torch.cuda.device_count() < world:
|
||||
self.skipTest(f"needs {world} GPUs")
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"torch.distributed.run",
|
||||
f"--nproc-per-node={world}",
|
||||
f"--master-port={port}",
|
||||
__file__,
|
||||
"--worker",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=1200,
|
||||
)
|
||||
print(proc.stdout[-4000:])
|
||||
if proc.returncode != 0:
|
||||
print(proc.stderr[-4000:], file=sys.stderr)
|
||||
self.assertEqual(proc.returncode, 0, "pipelined USPAttention diverged")
|
||||
if "SKIP" not in proc.stdout:
|
||||
self.assertIn(f"USP_PIPELINE_PARITY PASS world={world}", proc.stdout)
|
||||
|
||||
def test_two_ranks_match_sequential_bitwise(self):
|
||||
self._run(2, 29547)
|
||||
|
||||
def test_four_ranks_match_sequential_bitwise(self):
|
||||
self._run(4, 29557)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--worker" in sys.argv:
|
||||
raise SystemExit(_worker())
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user