[diffusion] perf: move MiniMax-H3's pipelined Ulysses copies off the SMs (#43361)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-10 08:58:07 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 9b37e28b83
commit bd7c7971ba
4 changed files with 179 additions and 74 deletions
@@ -23,7 +23,6 @@ import torch
import torch.distributed as dist
from sglang.kernels.ops.communication.ipc_a2a import load_ipc_a2a_sync
from sglang.kernels.ops.diffusion import pack_qkv_destination_major
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
_Unsupported,
@@ -50,6 +49,38 @@ def _member_devices(group, device: int) -> list[int]:
return devices
_CU_MEMORYTYPE_DEVICE = 2
def _memcpy2d_struct():
import ctypes
class Memcpy2D(ctypes.Structure): # CUDA_MEMCPY2D
_fields_ = [
("srcXInBytes", ctypes.c_size_t),
("srcY", ctypes.c_size_t),
("srcMemoryType", ctypes.c_int),
("srcHost", ctypes.c_void_p),
("srcDevice", ctypes.c_uint64),
("srcArray", ctypes.c_void_p),
("srcPitch", ctypes.c_size_t),
("dstXInBytes", ctypes.c_size_t),
("dstY", ctypes.c_size_t),
("dstMemoryType", ctypes.c_int),
("dstHost", ctypes.c_void_p),
("dstDevice", ctypes.c_uint64),
("dstArray", ctypes.c_void_p),
("dstPitch", ctypes.c_size_t),
("WidthInBytes", ctypes.c_size_t),
("Height", ctypes.c_size_t),
]
return Memcpy2D
_Memcpy2D = _memcpy2d_struct()
class _StreamMemOps:
"""Front-end signal and wait (cuStreamWriteValue32 / cuStreamWaitValue32).
@@ -69,6 +100,9 @@ class _StreamMemOps:
for fn in (self._write, self._wait):
fn.argtypes = args
fn.restype = ctypes.c_int
self._copy2d = cuda.cuMemcpy2DAsync_v2
self._copy2d.argtypes = [ctypes.POINTER(_Memcpy2D), ctypes.c_void_p]
self._copy2d.restype = ctypes.c_int
# drivers can disable stream memory operations; find out here, inside
# the group's agreed initialization, rather than mid-forward
flag = torch.zeros(1, dtype=torch.int32, device="cuda")
@@ -92,14 +126,41 @@ class _StreamMemOps:
if rc:
raise RuntimeError(f"cuStreamWaitValue32 failed with CUresult {rc}")
def copy2d(
self,
stream,
dst: int,
dst_pitch: int,
src: int,
src_pitch: int,
width: int,
height: int,
) -> None:
"""`height` rows of `width` bytes between pitched device pointers, on the
copy engine (a peer mapping is a device pointer like any other)."""
desc = _Memcpy2D(
srcMemoryType=_CU_MEMORYTYPE_DEVICE,
srcDevice=src,
srcPitch=src_pitch,
dstMemoryType=_CU_MEMORYTYPE_DEVICE,
dstDevice=dst,
dstPitch=dst_pitch,
WidthInBytes=width,
Height=height,
)
rc = self._copy2d(desc, stream.cuda_stream)
if rc:
raise RuntimeError(f"cuMemcpy2DAsync failed with CUresult {rc}")
class _PipelineBuffers:
"""Receive slots and flags for one pipelined-attention shape.
Input slots are group-major ([2, groups, world, chunk]) so a group's rows
form one [S, group_heads, 3 * head_dim] tensor for attention; output slots
are source-major ([2, world, groups, chunk]), each block a peer's rows for
one head group, copied into the merged output as it arrives.
Input slots are [2, groups, 3, world, chunk]: group g's q (k, v) rows from
every rank form one contiguous [S, group_heads, head_dim] tensor for
attention. Output slots are source-major ([2, world, groups, chunk]), each
block a peer's rows for one head group, copied into the merged output as it
arrives.
"""
def __init__(self, state, s_local, heads, head_dim, groups, dtype):
@@ -108,7 +169,7 @@ class _PipelineBuffers:
rows = s_local * self.group_heads
zeros = lambda *shape, dt=dtype: torch.zeros(*shape, dtype=dt, device="cuda")
self.inb = state._share(
zeros(2, groups, world, rows * 3 * head_dim), state.group
zeros(2, groups, 3, world, rows * head_dim), state.group
)
self.outb = state._share(zeros(2, world, groups, rows * head_dim), state.group)
# fin[p]: groups peer p has delivered; fout[g, p]: calls whose group g
@@ -116,11 +177,13 @@ class _PipelineBuffers:
# output counter could be lowered by a signal that lands late.
self.fin = state._share(zeros(world, dt=torch.int32), state.group)
self.fout = state._share(zeros(groups, world, dt=torch.int32), state.group)
# the caller's q/k for each peer block; v moves straight from its source
self.send = torch.empty(
world * groups,
2,
s_local,
self.group_heads,
3 * head_dim,
head_dim,
dtype=dtype,
device="cuda",
)
@@ -256,20 +319,22 @@ class IpcA2AMultiState:
q/k/v are [s_local, heads, head_dim], sequence-sharded with every head.
``attend(q, k, v)`` runs attention on [S, heads / (world * groups),
head_dim] views of one head group and must treat heads independently,
head_dim] tensors of one head group and must treat heads independently,
which keeps the result bit-identical to the sequential exchange.
Group g's copies run on the copy engine while group g-1 attends, and
each group attends on its own stream and sends its output from there,
so no stream waits on a kernel still running on another. Returns
[s_local, heads, head_dim], or None when this call cannot pipeline.
``groups < 0`` picks the count and steps aside, on every rank, when
the buffers would not fit.
so no stream waits on a kernel still running on another. Every copy,
the merge into the output included, is a pitched copy-engine copy, so
the exchange takes no SMs. Returns [s_local, heads, head_dim], or None
when this call cannot pipeline. ``groups < 0`` picks the count and
steps aside, on every rank, when the buffers would not fit.
``fill(head_start, head_count, dst)``, when given, writes those heads'
q/k/v into a [s_local, head_count, 3 * head_dim] block in place of the
pack (the caller's QK-norm writes its output there); blocks for this
rank go straight into its own receive slot. It is only called once the
call is known to pipeline.
``fill(head_start, head_count, q_dst, k_dst)``, when given, writes those
heads' q and k into two [s_local, head_count, head_dim] blocks in place
of a copy (the caller's QK-norm writes its output there); blocks for
this rank go straight into its own receive slot, and v always moves
straight from ``v``. It is only called once the call is known to
pipeline.
"""
world, r = self.world, self.rank
s_local, heads, head_dim = q.shape
@@ -278,10 +343,14 @@ class IpcA2AMultiState:
groups = next(
(n for n in _AUTO_PIPELINE_GROUPS if heads % (world * n) == 0), 0
)
# each token's heads of one group must be one contiguous run to move
# with a pitched copy
copied = (v,) if fill is not None else (q, k, v)
if (
not groups
or heads % (world * groups)
or not (q.shape == k.shape == v.shape)
or any(t.stride(2) != 1 or t.stride(1) != head_dim for t in copied)
):
return None
key = ("pipeline", s_local, heads, head_dim, groups, q.dtype)
@@ -289,8 +358,8 @@ class IpcA2AMultiState:
if bufs is None:
if torch.cuda.is_current_stream_capturing() or key in self.declined:
return None
# send + both receive and output slots: 11 * s_local * heads * head_dim
need = 11 * s_local * heads * head_dim * q.element_size()
# send + both receive and output slots: 10 * s_local * heads * head_dim
need = 10 * s_local * heads * head_dim * q.element_size()
if auto and not self._fits_on_every_rank(need):
logger.info(
"Ulysses pipeline buffers (%.1f GiB) do not fit; using the "
@@ -312,62 +381,82 @@ class IpcA2AMultiState:
bufs.calls += 1
call = bufs.calls
peers = [(r + step) % world for step in range(1, world)]
own = []
if fill is None:
# block p * groups + g is contiguous: group g of the heads rank p owns
send = pack_qkv_destination_major(q, k, v, world * groups, out=bufs.send)
cin.wait_stream(main)
with torch.cuda.stream(cin):
for g in range(groups):
bufs.inb[r][slot, g, r].copy_(
send[r * groups + g].view(-1), non_blocking=True
)
own.append(cin.record_event())
for p in peers:
bufs.inb[p][slot, g, r].copy_(
send[p * groups + g].view(-1), non_blocking=True
)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
else:
# Group by group: the peers' blocks first, so their copies start
# while this rank fills its own block straight into its slot.
send = bufs.send
block = lambda t: t.view(s_local, hg, 3 * head_dim)
sent = []
esz = q.element_size()
width = hg * head_dim * esz # one token's heads of one group
chunk = s_local * width # one rank's rows of one group, one of q/k/v
start = lambda p, g: (p * groups + g) * hg # first head of block (p, g)
slot_of = lambda p, g, part: bufs.inb[p][slot, g, part, r] # my rows there
def send_rows(stream, t, p, g, part):
src = t[:, start(p, g)]
mem.copy2d(
stream,
slot_of(p, g, part).data_ptr(),
width,
src.data_ptr(),
t.stride(0) * esz,
width,
s_local,
)
ready = main.record_event()
own, sent = [], []
if fill is not None:
# group by group: the peers' blocks first, so their copies start
# while this rank fills its own block straight into its slot
block = lambda t: t.view(s_local, hg, head_dim)
for g in range(groups):
for p in peers:
fill((p * groups + g) * hg, hg, block(send[p * groups + g]))
qk = bufs.send[p * groups + g]
fill(start(p, g), hg, qk[0], qk[1])
sent.append(main.record_event())
fill((r * groups + g) * hg, hg, block(bufs.inb[r][slot, g, r]))
fill(start(r, g), hg, block(slot_of(r, g, 0)), block(slot_of(r, g, 1)))
own.append(main.record_event())
with torch.cuda.stream(cin):
for g in range(groups):
own_cin = []
cin.wait_event(ready)
with torch.cuda.stream(cin):
for g in range(groups):
for p in (r, *peers):
send_rows(cin, v, p, g, 2)
if fill is None:
send_rows(cin, q, p, g, 0)
send_rows(cin, k, p, g, 1)
if p == r:
own_cin.append(cin.record_event())
if fill is not None:
cin.wait_event(sent[g])
for p in peers:
bufs.inb[p][slot, g, r].copy_(
send[p * groups + g].view(-1), non_blocking=True
# q then k: two rows of `chunk`, one part apart in the slot
mem.copy2d(
cin,
slot_of(p, g, 0).data_ptr(),
slot_of(p, g, 1).data_ptr() - slot_of(p, g, 0).data_ptr(),
bufs.send[p * groups + g].data_ptr(),
chunk,
chunk,
2,
)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
for p in peers:
mem.write(cin, bufs.fin[p].narrow(0, r, 1), base + g + 1)
# Head block b = p * groups + g of the output is group g of rank p's
# heads. Each group stream writes its blocks as soon as they exist, so
# the merge overlaps the groups still attending instead of trailing them.
merged = torch.empty(s_local, heads, head_dim, dtype=q.dtype, device=q.device)
merged_blocks = merged.view(s_local, world * groups, hg, head_dim)
merged_pitch = heads * head_dim * esz
done = []
for g in range(groups):
stream = self.pipe_group_streams[g]
stream.wait_event(own[g])
if fill is not None:
stream.wait_event(own[g])
stream.wait_event(own_cin[g])
with torch.cuda.stream(stream):
for p in peers:
mem.wait(stream, bufs.fin[r].narrow(0, p, 1), base + g + 1)
qkv = bufs.inb[r][slot, g].view(world * s_local, hg, 3 * head_dim)
out = attend(
qkv[..., :head_dim],
qkv[..., head_dim : 2 * head_dim],
qkv[..., 2 * head_dim :],
part = lambda i: bufs.inb[r][slot, g, i].view(
world * s_local, hg, head_dim
)
out = attend(part(0), part(1), part(2)).contiguous()
rows = lambda p: out[p * s_local : (p + 1) * s_local]
for p in peers:
bufs.outb[p][slot, r, g].copy_(
@@ -375,16 +464,31 @@ class IpcA2AMultiState:
)
for p in peers:
mem.write(stream, bufs.fout[p][g].narrow(0, r, 1), call)
merged_blocks[:, r * groups + g].copy_(rows(r))
mem.copy2d(
stream,
merged_blocks[:, r * groups + g].data_ptr(),
merged_pitch,
rows(r).data_ptr(),
width,
width,
s_local,
)
for p in peers:
mem.wait(stream, bufs.fout[r][g].narrow(0, p, 1), call)
merged_blocks[:, p * groups + g].copy_(
bufs.outb[r][slot, p, g].view(s_local, hg, head_dim)
mem.copy2d(
stream,
merged_blocks[:, p * groups + g].data_ptr(),
merged_pitch,
bufs.outb[r][slot, p, g].data_ptr(),
width,
width,
s_local,
)
done.append(stream.record_event())
for event in done:
main.wait_event(event)
# the next call's pack rewrites `send`, which the copy stream reads
# the next call's fill rewrites `send` and the caller may free `v`, both
# read by the copy stream
main.wait_stream(cin)
return merged
@@ -737,9 +737,10 @@ def _minimax_h3_qknorm_rope_pipelined_attention(
"""The pipelined attention with QK-norm + RoPE writing into the exchange.
q/k are the raw projections. Each destination block gets its heads'
normalized q/k and v written in place, so the in-place norm and the pack
become one pass. Same kernel arithmetic, so the result is bit-identical.
Returns None, having written nothing, when the call cannot pipeline.
normalized q and k written in place, so the in-place norm and the pack
become one pass, and v moves on the copy engine. Same kernel arithmetic,
so the result is bit-identical. Returns None, having written nothing,
when the call cannot pipeline.
"""
# under graph capture the forward keeps the in-place norm, and the pipeline
# runs from the attention core's eager break point instead
@@ -753,13 +754,15 @@ def _minimax_h3_qknorm_rope_pipelined_attention(
head_dim = attention.head_dim
q_weight, k_weight = attention.q_norm.weight, attention.k_norm.weight
def fill(head_start: int, head_count: int, dst: torch.Tensor) -> None:
def fill(
head_start: int, head_count: int, q_dst: torch.Tensor, k_dst: torch.Tensor
) -> None:
heads = slice(head_start, head_start + head_count)
fused_qknorm_rope_out_of_place(
q[:, heads],
k[:, heads],
dst[..., :head_dim],
dst[..., head_dim : 2 * head_dim],
q_dst,
k_dst,
q_weight,
k_weight,
cos_sin_cache,
@@ -770,7 +773,6 @@ def _minimax_h3_qknorm_rope_pipelined_attention(
rope_dim=cos_sin_cache.shape[-1],
round_norm_before_rope=True,
)
dst[..., 2 * head_dim :].copy_(v[:, heads])
return _minimax_h3_pipelined_dense_attention(
attention,
@@ -96,8 +96,8 @@ def _worker() -> int:
f"pipelined {(s_local, heads, head_dim, groups)} call {call}"
)
# fill: the caller writes each destination block itself (H3's QK-norm does),
# own blocks straight into this rank's receive slot
# fill: the caller writes each destination's q/k itself (H3's QK-norm does),
# own blocks straight into this rank's receive slot; v moves on its own
def transform(t, scale):
return (t.float() * scale + 1).to(t.dtype)
@@ -116,11 +116,10 @@ def _worker() -> int:
]
q, k, v = (t.narrow(0, rank * s_local, s_local).contiguous() for t in full)
def fill(head_start, head_count, dst):
def fill(head_start, head_count, q_dst, k_dst):
h = slice(head_start, head_start + head_count)
dst[..., :head_dim].copy_(transform(q[:, h], 2))
dst[..., head_dim : 2 * head_dim].copy_(transform(k[:, h], 3))
dst[..., 2 * head_dim :].copy_(v[:, h])
q_dst.copy_(transform(q[:, h], 2))
k_dst.copy_(transform(k[:, h], 3))
ref = sequential(transform(q, 2), transform(k, 3), v)
got = ulysses_pipelined_attention(q, k, v, _attend, groups, fill=fill)