[Perf] [NVIDIA] custom-all-reduce: Add 2shot_push all-reduce and retune SM100 TP4/TP8 (#42811)

This commit is contained in:
Chunan Zeng
2026-10-16 13:43:18 -07:00
committed by GitHub
parent 21791e7023
commit 07950be745
4 changed files with 93 additions and 55 deletions
@@ -136,7 +136,7 @@ struct MultiCastImpl {
#define ALL_REDUCE_KERNEL __global__ __launch_bounds__(1024, 1)
template <typename Impl, typename T, uint32_t kWorldSize, bool kUsePDL>
template <typename T, uint32_t kWorldSize, bool kUsePDL>
ALL_REDUCE_KERNEL void all_reduce_1shot_push_kernel(const __grid_constant__ AllReducePushParams<kWorldSize> params) {
using namespace device;
constexpr uint32_t kVecSize = 16 / (sizeof(T) * 2);
@@ -224,25 +224,26 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
using Lamport = distributed::LamportTrait<T, kVecSize * 2, /*kAtom=*/4>;
const auto r = params.rank;
const auto num_vecs = params.num_vecs;
const auto chunk_vecs = (num_vecs + kWorldSize - 1) / kWorldSize;
const auto chunk_len = [&](uint32_t c) { return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs)); };
const auto chunk_vecs = div_ceil(num_vecs, kWorldSize);
const auto get_chunk_len = [&](uint32_t c) { //
return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs));
};
const auto num_threads = blockDim.x * gridDim.x;
const auto num_warps = num_threads / kWarpThreads;
const auto warp_in_block = threadIdx.x / kWarpThreads;
const auto lane_id = threadIdx.x % kWarpThreads;
const auto global_warp_id = blockIdx.x + gridDim.x * warp_in_block;
const auto global_tid = global_warp_id * kWarpThreads + lane_id;
PDLWaitPrimary<kUsePDL>();
const auto epoch = distributed::PushEpoch<kWorldSize>{params.ws};
// Each warp serves one peer, starting at r + 1 so every peer's ingress is
// busy from the first wave and the slot pointer stays warp-uniform.
auto peer = r + 1 + global_warp_id % kFan;
if (peer >= kWorldSize) peer -= kWorldSize;
// peer will cover 0..kWorldSize - 1, except for local rank `r`
const auto peer = (r + 1 + global_warp_id % kFan) % kWorldSize;
const auto group_warps = num_warps / kFan + (global_warp_id % kFan < num_warps % kFan ? 1 : 0);
const auto group_tid = (global_warp_id / kFan) * kWarpThreads + lane_id;
const auto group_stride = group_warps * kWarpThreads;
const auto peer_len = chunk_len(peer);
const auto peer_len = get_chunk_len(peer);
PDLWaitPrimary<kUsePDL>();
const auto epoch = distributed::PushEpoch<kWorldSize>{params.ws};
// reduce-scatter: my partial of the peer's chunk to its owner
{
@@ -265,17 +266,12 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
}
// reduce my chunk, keep it and fan it out
const auto my_len = chunk_len(r);
const auto my_len = get_chunk_len(r);
for (auto j = global_tid; j < my_len; j += num_threads) {
const auto vid = r * chunk_vecs + j;
vec_t vec[kWorldSize];
#pragma unroll
for (uint32_t i = 0; i < kWorldSize; ++i) {
if (i == r) {
vec[i].load(params.input, vid);
Lamport::clear_pos_zero(vec[i].data());
}
}
vec_t local_vec;
local_vec.load(params.input, vid);
do {
bool has_zero = false;
#pragma unroll
@@ -284,10 +280,14 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
}
#pragma unroll
for (uint32_t i = 0; i < kWorldSize; ++i) {
if (i != r) has_zero |= Lamport::has_pos_zero(vec[i].data());
has_zero |= ((i != r) & Lamport::has_pos_zero(vec[i].data()));
}
if (!has_zero) break;
} while (true);
#pragma unroll
for (uint32_t i = 0; i < kWorldSize; ++i) {
if (i == r) vec[i] = local_vec;
}
auto out_vec = reduce_vec(vec);
// remap before the local store too, so every rank holds identical bits
Lamport::clear_pos_zero(out_vec.data());
@@ -451,13 +451,12 @@ struct AllReduceKernel {
.rank = push.rank,
.ws = push.get_workspace<kWorldSize>(two_shot ? int64_t{2} * chunk_vecs * 16 : nbytes),
};
using Impl = LoadStoreImpl<vec_t, kWorldSize, /*kUseGraph=*/false>;
const auto kernel = two_shot ? all_reduce_2shot_push_kernel<T, kWorldSize, kUsePDL>
: all_reduce_1shot_push_kernel<Impl, T, kWorldSize, kUsePDL>;
: all_reduce_1shot_push_kernel<T, kWorldSize, kUsePDL>;
// the grid is bound to the counter array and must stay constant
auto block_size = choose_block_size(num_vecs);
if (two_shot) {
// Every peer needs a warp, including when callers use a small grid.
// at least `kWorldSize - 1` warps in all, 1 for each peer
block_size = std::max(block_size, 32u * div_ceil(kWorldSize - 1, push.num_blocks));
}
LaunchKernel(push.num_blocks, block_size, stream) //
@@ -31,26 +31,36 @@ class Range(NamedTuple):
class Heuristic(NamedTuple):
"""Self-contained algo ranges for one dispatch context (graph or eager).
Four algos are tried in order of preference (fastest first):
Five algos are tried in order of preference (fastest first):
1. ``1shot_push``: nbytes <= ``one_shot_push_threshold``
2. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold``
3. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes``
2. ``2shot_push``: nbytes <= ``two_shot_push_threshold``
3. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold``
4. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes``
(only when multicast is enabled at runtime)
4. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold``
5. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold``
Above all of these, the caller falls back to NCCL.
Setting two adjacent thresholds equal effectively disables the middle
algo; leaving ``mc`` at the default disables multicast.
algo; leaving ``mc`` / ``two_shot_push_threshold`` at the default
disables that algo.
The field order below is the positional layout of the table rows
(``1shot_push, 1shot_pull, 2shot_pull, mc``); ``two_shot_push_threshold``
is set by keyword, see ``_sm100_configs``.
"""
one_shot_push_threshold: int
one_shot_pull_threshold: int
two_shot_pull_threshold: int
mc: Range = Range(0, 0) # default: multicast disabled in this context
two_shot_push_threshold: int = 0 # default: 2shot_push disabled
@property
def max_push_bytes(self) -> int:
return self.one_shot_push_threshold
def max_push_bytes(self, world_size: int) -> int:
"""Per-slot push plane bytes this heuristic needs."""
return max(
self.one_shot_push_threshold,
two_shot_push_slot_bytes(self.two_shot_push_threshold, world_size),
)
@property
def max_pull_bytes(self) -> int:
@@ -62,15 +72,32 @@ class Heuristic(NamedTuple):
self.mc.max_bytes,
)
def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "Heuristic":
def clip(
self, *, max_push_bytes: int, max_pull_bytes: int, world_size: int
) -> "Heuristic":
return Heuristic(
min(self.one_shot_push_threshold, max_push_bytes),
min(self.one_shot_pull_threshold, max_pull_bytes),
min(self.two_shot_pull_threshold, max_pull_bytes),
self.mc.clip(max_pull_bytes),
one_shot_push_threshold=min(self.one_shot_push_threshold, max_push_bytes),
one_shot_pull_threshold=min(self.one_shot_pull_threshold, max_pull_bytes),
two_shot_pull_threshold=min(self.two_shot_pull_threshold, max_pull_bytes),
mc=self.mc.clip(max_pull_bytes),
two_shot_push_threshold=min(
self.two_shot_push_threshold,
two_shot_push_max_bytes(max_push_bytes, world_size),
),
)
def two_shot_push_slot_bytes(nbytes: int, world_size: int) -> int:
"""A 2shot_push slot holds a partial half and a gathered half of one
``ceil(nbytes / world_size)`` chunk, in 16 B vectors."""
return 32 * -(-nbytes // (16 * world_size))
def two_shot_push_max_bytes(slot_bytes: int, world_size: int) -> int:
"""Inverse of ``two_shot_push_slot_bytes``: the largest tensor a slot fits."""
return world_size * 16 * (slot_bytes // 32)
class AllReduceConfig(NamedTuple):
"""All tuning knobs for a single (arch, world_size).
@@ -88,22 +115,19 @@ class AllReduceConfig(NamedTuple):
num_pull_blocks: int
num_mc_blocks: Optional[int]
@property
def max_push_bytes(self) -> int:
return max(self.graph.max_push_bytes, self.eager.max_push_bytes)
def max_push_bytes(self, world_size: int) -> int:
return max(
self.graph.max_push_bytes(world_size),
self.eager.max_push_bytes(world_size),
)
@property
def max_pull_bytes(self) -> int:
return max(self.graph.max_pull_bytes, self.eager.max_pull_bytes)
def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "AllReduceConfig":
def clip(self, **kwargs) -> "AllReduceConfig":
return self._replace(
graph=self.graph.clip(
max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes
),
eager=self.eager.clip(
max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes
),
graph=self.graph.clip(**kwargs), eager=self.eager.clip(**kwargs)
)
@@ -112,15 +136,22 @@ def _pack_heuristic(*args) -> Heuristic:
return Heuristic(*arg_list)
# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200.
# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200;
# the 2shot_push rows (world 4 / 8) on B300.
@cache
def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
mc_blocks = {5: 64, 6: 48, 7: 48, 8: 32, 16: 32}
def config(world_size: int, *, graph: tuple, eager: tuple) -> AllReduceConfig:
def config(
world_size: int, *, graph: tuple, eager: tuple, two_shot_push: int = 0
) -> AllReduceConfig:
return AllReduceConfig(
graph=_pack_heuristic(*graph),
eager=_pack_heuristic(*eager),
graph=_pack_heuristic(*graph)._replace(
two_shot_push_threshold=two_shot_push
),
eager=_pack_heuristic(*eager)._replace(
two_shot_push_threshold=two_shot_push
),
num_push_blocks=num_sm,
num_pull_blocks=num_sm if world_size == 2 else 96,
num_mc_blocks=mc_blocks.get(world_size),
@@ -139,8 +170,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
),
4: config(
4,
graph=(2.250 * MB, 2.250 * MB, 128.0 * MB),
eager=(3.000 * MB, 3.000 * MB, 32.00 * MB),
graph=(0.625 * MB, 0.625 * MB, 128.0 * MB),
eager=(0.625 * MB, 0.625 * MB, 32.00 * MB),
two_shot_push=8 * MB,
),
5: config(
5,
@@ -159,8 +191,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
),
8: config(
8,
graph=(0.500 * MB, 0.500 * MB, 128.0 * MB, Range(8 * MB, 128 * MB)),
eager=(0.750 * MB, 0.750 * MB, 128.0 * MB, Range(0, 128 * MB)),
graph=(256 * KB, 256 * KB, 128.0 * MB, Range(8 * MB, 128 * MB)),
eager=(256 * KB, 256 * KB, 128.0 * MB, Range(0, 128 * MB)),
two_shot_push=8 * MB,
),
16: config(
16,
@@ -145,7 +145,7 @@ class CustomAllReduceV2:
if max_pull_size is None:
max_pull_size = min(base_config.max_pull_bytes, max_size)
if max_push_size is None:
max_push_size = min(base_config.max_push_bytes, max_size)
max_push_size = min(base_config.max_push_bytes(self.world_size), max_size)
if _FORCE_PULL_SIZE_KB is not None:
max_pull_size = int(_FORCE_PULL_SIZE_KB) * 1024
if _FORCE_PUSH_SIZE_KB is not None:
@@ -185,7 +185,9 @@ class CustomAllReduceV2:
if max_push_blocks is not None:
num_push_blocks = max(max_push_blocks, 1)
self.config = base_config.clip(
max_push_bytes=self.max_push_size, max_pull_bytes=self.max_pull_size
max_push_bytes=self.max_push_size,
max_pull_bytes=self.max_pull_size,
world_size=self.world_size,
)._replace(num_pull_blocks=num_pull_blocks, num_push_blocks=num_push_blocks)
self.override_algo: Optional[AllReduceAlgo] = None
# On a multi-node (MNNVL) group the symm-mem workspace plane works
@@ -338,6 +340,8 @@ class CustomAllReduceV2:
can_use_multicast = self.config.num_mc_blocks is not None
if nbytes <= heuristic.one_shot_push_threshold:
return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PUSH)
if nbytes <= heuristic.two_shot_push_threshold:
return AllReduceConfig(AllReduceAlgo.TWO_SHOT_PUSH)
if nbytes <= heuristic.one_shot_pull_threshold:
return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PULL, use_graph=can_use_graph)
if can_use_multicast and heuristic.mc.contains(nbytes):