[Perf] [NVIDIA] custom-all-reduce: Add 2shot_push all-reduce and retune SM100 TP4/TP8 (#42811)
This commit is contained in:
@@ -136,7 +136,7 @@ struct MultiCastImpl {
|
||||
|
||||
#define ALL_REDUCE_KERNEL __global__ __launch_bounds__(1024, 1)
|
||||
|
||||
template <typename Impl, typename T, uint32_t kWorldSize, bool kUsePDL>
|
||||
template <typename T, uint32_t kWorldSize, bool kUsePDL>
|
||||
ALL_REDUCE_KERNEL void all_reduce_1shot_push_kernel(const __grid_constant__ AllReducePushParams<kWorldSize> params) {
|
||||
using namespace device;
|
||||
constexpr uint32_t kVecSize = 16 / (sizeof(T) * 2);
|
||||
@@ -224,25 +224,26 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
|
||||
using Lamport = distributed::LamportTrait<T, kVecSize * 2, /*kAtom=*/4>;
|
||||
const auto r = params.rank;
|
||||
const auto num_vecs = params.num_vecs;
|
||||
const auto chunk_vecs = (num_vecs + kWorldSize - 1) / kWorldSize;
|
||||
const auto chunk_len = [&](uint32_t c) { return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs)); };
|
||||
const auto chunk_vecs = div_ceil(num_vecs, kWorldSize);
|
||||
const auto get_chunk_len = [&](uint32_t c) { //
|
||||
return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs));
|
||||
};
|
||||
const auto num_threads = blockDim.x * gridDim.x;
|
||||
const auto num_warps = num_threads / kWarpThreads;
|
||||
const auto warp_in_block = threadIdx.x / kWarpThreads;
|
||||
const auto lane_id = threadIdx.x % kWarpThreads;
|
||||
const auto global_warp_id = blockIdx.x + gridDim.x * warp_in_block;
|
||||
const auto global_tid = global_warp_id * kWarpThreads + lane_id;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto epoch = distributed::PushEpoch<kWorldSize>{params.ws};
|
||||
|
||||
// Each warp serves one peer, starting at r + 1 so every peer's ingress is
|
||||
// busy from the first wave and the slot pointer stays warp-uniform.
|
||||
auto peer = r + 1 + global_warp_id % kFan;
|
||||
if (peer >= kWorldSize) peer -= kWorldSize;
|
||||
// peer will cover 0..kWorldSize - 1, except for local rank `r`
|
||||
const auto peer = (r + 1 + global_warp_id % kFan) % kWorldSize;
|
||||
const auto group_warps = num_warps / kFan + (global_warp_id % kFan < num_warps % kFan ? 1 : 0);
|
||||
const auto group_tid = (global_warp_id / kFan) * kWarpThreads + lane_id;
|
||||
const auto group_stride = group_warps * kWarpThreads;
|
||||
const auto peer_len = chunk_len(peer);
|
||||
const auto peer_len = get_chunk_len(peer);
|
||||
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
const auto epoch = distributed::PushEpoch<kWorldSize>{params.ws};
|
||||
|
||||
// reduce-scatter: my partial of the peer's chunk to its owner
|
||||
{
|
||||
@@ -265,17 +266,12 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
|
||||
}
|
||||
|
||||
// reduce my chunk, keep it and fan it out
|
||||
const auto my_len = chunk_len(r);
|
||||
const auto my_len = get_chunk_len(r);
|
||||
for (auto j = global_tid; j < my_len; j += num_threads) {
|
||||
const auto vid = r * chunk_vecs + j;
|
||||
vec_t vec[kWorldSize];
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kWorldSize; ++i) {
|
||||
if (i == r) {
|
||||
vec[i].load(params.input, vid);
|
||||
Lamport::clear_pos_zero(vec[i].data());
|
||||
}
|
||||
}
|
||||
vec_t local_vec;
|
||||
local_vec.load(params.input, vid);
|
||||
do {
|
||||
bool has_zero = false;
|
||||
#pragma unroll
|
||||
@@ -284,10 +280,14 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR
|
||||
}
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kWorldSize; ++i) {
|
||||
if (i != r) has_zero |= Lamport::has_pos_zero(vec[i].data());
|
||||
has_zero |= ((i != r) & Lamport::has_pos_zero(vec[i].data()));
|
||||
}
|
||||
if (!has_zero) break;
|
||||
} while (true);
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < kWorldSize; ++i) {
|
||||
if (i == r) vec[i] = local_vec;
|
||||
}
|
||||
auto out_vec = reduce_vec(vec);
|
||||
// remap before the local store too, so every rank holds identical bits
|
||||
Lamport::clear_pos_zero(out_vec.data());
|
||||
@@ -451,13 +451,12 @@ struct AllReduceKernel {
|
||||
.rank = push.rank,
|
||||
.ws = push.get_workspace<kWorldSize>(two_shot ? int64_t{2} * chunk_vecs * 16 : nbytes),
|
||||
};
|
||||
using Impl = LoadStoreImpl<vec_t, kWorldSize, /*kUseGraph=*/false>;
|
||||
const auto kernel = two_shot ? all_reduce_2shot_push_kernel<T, kWorldSize, kUsePDL>
|
||||
: all_reduce_1shot_push_kernel<Impl, T, kWorldSize, kUsePDL>;
|
||||
: all_reduce_1shot_push_kernel<T, kWorldSize, kUsePDL>;
|
||||
// the grid is bound to the counter array and must stay constant
|
||||
auto block_size = choose_block_size(num_vecs);
|
||||
if (two_shot) {
|
||||
// Every peer needs a warp, including when callers use a small grid.
|
||||
// at least `kWorldSize - 1` warps in all, 1 for each peer
|
||||
block_size = std::max(block_size, 32u * div_ceil(kWorldSize - 1, push.num_blocks));
|
||||
}
|
||||
LaunchKernel(push.num_blocks, block_size, stream) //
|
||||
|
||||
+64
-31
@@ -31,26 +31,36 @@ class Range(NamedTuple):
|
||||
class Heuristic(NamedTuple):
|
||||
"""Self-contained algo ranges for one dispatch context (graph or eager).
|
||||
|
||||
Four algos are tried in order of preference (fastest first):
|
||||
Five algos are tried in order of preference (fastest first):
|
||||
1. ``1shot_push``: nbytes <= ``one_shot_push_threshold``
|
||||
2. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold``
|
||||
3. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes``
|
||||
2. ``2shot_push``: nbytes <= ``two_shot_push_threshold``
|
||||
3. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold``
|
||||
4. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes``
|
||||
(only when multicast is enabled at runtime)
|
||||
4. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold``
|
||||
5. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold``
|
||||
Above all of these, the caller falls back to NCCL.
|
||||
|
||||
Setting two adjacent thresholds equal effectively disables the middle
|
||||
algo; leaving ``mc`` at the default disables multicast.
|
||||
algo; leaving ``mc`` / ``two_shot_push_threshold`` at the default
|
||||
disables that algo.
|
||||
|
||||
The field order below is the positional layout of the table rows
|
||||
(``1shot_push, 1shot_pull, 2shot_pull, mc``); ``two_shot_push_threshold``
|
||||
is set by keyword, see ``_sm100_configs``.
|
||||
"""
|
||||
|
||||
one_shot_push_threshold: int
|
||||
one_shot_pull_threshold: int
|
||||
two_shot_pull_threshold: int
|
||||
mc: Range = Range(0, 0) # default: multicast disabled in this context
|
||||
two_shot_push_threshold: int = 0 # default: 2shot_push disabled
|
||||
|
||||
@property
|
||||
def max_push_bytes(self) -> int:
|
||||
return self.one_shot_push_threshold
|
||||
def max_push_bytes(self, world_size: int) -> int:
|
||||
"""Per-slot push plane bytes this heuristic needs."""
|
||||
return max(
|
||||
self.one_shot_push_threshold,
|
||||
two_shot_push_slot_bytes(self.two_shot_push_threshold, world_size),
|
||||
)
|
||||
|
||||
@property
|
||||
def max_pull_bytes(self) -> int:
|
||||
@@ -62,15 +72,32 @@ class Heuristic(NamedTuple):
|
||||
self.mc.max_bytes,
|
||||
)
|
||||
|
||||
def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "Heuristic":
|
||||
def clip(
|
||||
self, *, max_push_bytes: int, max_pull_bytes: int, world_size: int
|
||||
) -> "Heuristic":
|
||||
return Heuristic(
|
||||
min(self.one_shot_push_threshold, max_push_bytes),
|
||||
min(self.one_shot_pull_threshold, max_pull_bytes),
|
||||
min(self.two_shot_pull_threshold, max_pull_bytes),
|
||||
self.mc.clip(max_pull_bytes),
|
||||
one_shot_push_threshold=min(self.one_shot_push_threshold, max_push_bytes),
|
||||
one_shot_pull_threshold=min(self.one_shot_pull_threshold, max_pull_bytes),
|
||||
two_shot_pull_threshold=min(self.two_shot_pull_threshold, max_pull_bytes),
|
||||
mc=self.mc.clip(max_pull_bytes),
|
||||
two_shot_push_threshold=min(
|
||||
self.two_shot_push_threshold,
|
||||
two_shot_push_max_bytes(max_push_bytes, world_size),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def two_shot_push_slot_bytes(nbytes: int, world_size: int) -> int:
|
||||
"""A 2shot_push slot holds a partial half and a gathered half of one
|
||||
``ceil(nbytes / world_size)`` chunk, in 16 B vectors."""
|
||||
return 32 * -(-nbytes // (16 * world_size))
|
||||
|
||||
|
||||
def two_shot_push_max_bytes(slot_bytes: int, world_size: int) -> int:
|
||||
"""Inverse of ``two_shot_push_slot_bytes``: the largest tensor a slot fits."""
|
||||
return world_size * 16 * (slot_bytes // 32)
|
||||
|
||||
|
||||
class AllReduceConfig(NamedTuple):
|
||||
"""All tuning knobs for a single (arch, world_size).
|
||||
|
||||
@@ -88,22 +115,19 @@ class AllReduceConfig(NamedTuple):
|
||||
num_pull_blocks: int
|
||||
num_mc_blocks: Optional[int]
|
||||
|
||||
@property
|
||||
def max_push_bytes(self) -> int:
|
||||
return max(self.graph.max_push_bytes, self.eager.max_push_bytes)
|
||||
def max_push_bytes(self, world_size: int) -> int:
|
||||
return max(
|
||||
self.graph.max_push_bytes(world_size),
|
||||
self.eager.max_push_bytes(world_size),
|
||||
)
|
||||
|
||||
@property
|
||||
def max_pull_bytes(self) -> int:
|
||||
return max(self.graph.max_pull_bytes, self.eager.max_pull_bytes)
|
||||
|
||||
def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "AllReduceConfig":
|
||||
def clip(self, **kwargs) -> "AllReduceConfig":
|
||||
return self._replace(
|
||||
graph=self.graph.clip(
|
||||
max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes
|
||||
),
|
||||
eager=self.eager.clip(
|
||||
max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes
|
||||
),
|
||||
graph=self.graph.clip(**kwargs), eager=self.eager.clip(**kwargs)
|
||||
)
|
||||
|
||||
|
||||
@@ -112,15 +136,22 @@ def _pack_heuristic(*args) -> Heuristic:
|
||||
return Heuristic(*arg_list)
|
||||
|
||||
|
||||
# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200.
|
||||
# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200;
|
||||
# the 2shot_push rows (world 4 / 8) on B300.
|
||||
@cache
|
||||
def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
|
||||
mc_blocks = {5: 64, 6: 48, 7: 48, 8: 32, 16: 32}
|
||||
|
||||
def config(world_size: int, *, graph: tuple, eager: tuple) -> AllReduceConfig:
|
||||
def config(
|
||||
world_size: int, *, graph: tuple, eager: tuple, two_shot_push: int = 0
|
||||
) -> AllReduceConfig:
|
||||
return AllReduceConfig(
|
||||
graph=_pack_heuristic(*graph),
|
||||
eager=_pack_heuristic(*eager),
|
||||
graph=_pack_heuristic(*graph)._replace(
|
||||
two_shot_push_threshold=two_shot_push
|
||||
),
|
||||
eager=_pack_heuristic(*eager)._replace(
|
||||
two_shot_push_threshold=two_shot_push
|
||||
),
|
||||
num_push_blocks=num_sm,
|
||||
num_pull_blocks=num_sm if world_size == 2 else 96,
|
||||
num_mc_blocks=mc_blocks.get(world_size),
|
||||
@@ -139,8 +170,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
|
||||
),
|
||||
4: config(
|
||||
4,
|
||||
graph=(2.250 * MB, 2.250 * MB, 128.0 * MB),
|
||||
eager=(3.000 * MB, 3.000 * MB, 32.00 * MB),
|
||||
graph=(0.625 * MB, 0.625 * MB, 128.0 * MB),
|
||||
eager=(0.625 * MB, 0.625 * MB, 32.00 * MB),
|
||||
two_shot_push=8 * MB,
|
||||
),
|
||||
5: config(
|
||||
5,
|
||||
@@ -159,8 +191,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]:
|
||||
),
|
||||
8: config(
|
||||
8,
|
||||
graph=(0.500 * MB, 0.500 * MB, 128.0 * MB, Range(8 * MB, 128 * MB)),
|
||||
eager=(0.750 * MB, 0.750 * MB, 128.0 * MB, Range(0, 128 * MB)),
|
||||
graph=(256 * KB, 256 * KB, 128.0 * MB, Range(8 * MB, 128 * MB)),
|
||||
eager=(256 * KB, 256 * KB, 128.0 * MB, Range(0, 128 * MB)),
|
||||
two_shot_push=8 * MB,
|
||||
),
|
||||
16: config(
|
||||
16,
|
||||
|
||||
@@ -145,7 +145,7 @@ class CustomAllReduceV2:
|
||||
if max_pull_size is None:
|
||||
max_pull_size = min(base_config.max_pull_bytes, max_size)
|
||||
if max_push_size is None:
|
||||
max_push_size = min(base_config.max_push_bytes, max_size)
|
||||
max_push_size = min(base_config.max_push_bytes(self.world_size), max_size)
|
||||
if _FORCE_PULL_SIZE_KB is not None:
|
||||
max_pull_size = int(_FORCE_PULL_SIZE_KB) * 1024
|
||||
if _FORCE_PUSH_SIZE_KB is not None:
|
||||
@@ -185,7 +185,9 @@ class CustomAllReduceV2:
|
||||
if max_push_blocks is not None:
|
||||
num_push_blocks = max(max_push_blocks, 1)
|
||||
self.config = base_config.clip(
|
||||
max_push_bytes=self.max_push_size, max_pull_bytes=self.max_pull_size
|
||||
max_push_bytes=self.max_push_size,
|
||||
max_pull_bytes=self.max_pull_size,
|
||||
world_size=self.world_size,
|
||||
)._replace(num_pull_blocks=num_pull_blocks, num_push_blocks=num_push_blocks)
|
||||
self.override_algo: Optional[AllReduceAlgo] = None
|
||||
# On a multi-node (MNNVL) group the symm-mem workspace plane works
|
||||
@@ -338,6 +340,8 @@ class CustomAllReduceV2:
|
||||
can_use_multicast = self.config.num_mc_blocks is not None
|
||||
if nbytes <= heuristic.one_shot_push_threshold:
|
||||
return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PUSH)
|
||||
if nbytes <= heuristic.two_shot_push_threshold:
|
||||
return AllReduceConfig(AllReduceAlgo.TWO_SHOT_PUSH)
|
||||
if nbytes <= heuristic.one_shot_pull_threshold:
|
||||
return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PULL, use_graph=can_use_graph)
|
||||
if can_use_multicast and heuristic.mc.contains(nbytes):
|
||||
|
||||
Reference in New Issue
Block a user