From 07950be745136dc90de8a6f7f61d6fc734ae1858 Mon Sep 17 00:00:00 2001 From: Chunan Zeng Date: Fri, 16 Oct 2026 13:43:18 -0700 Subject: [PATCH] [Perf] [NVIDIA] custom-all-reduce: Add 2shot_push all-reduce and retune SM100 TP4/TP8 (#42811) --- .../csrc/distributed/custom_all_reduce.cuh | 43 ++++----- .../configs/custom_all_reduce_v2.py | 95 +++++++++++++------ .../custom_all_reduce_v2.py | 8 +- .../communication/test_custom_all_reduce.py | 2 + 4 files changed, 93 insertions(+), 55 deletions(-) diff --git a/python/sglang/kernels/jit/csrc/distributed/custom_all_reduce.cuh b/python/sglang/kernels/jit/csrc/distributed/custom_all_reduce.cuh index 481f84593f..724d80f868 100644 --- a/python/sglang/kernels/jit/csrc/distributed/custom_all_reduce.cuh +++ b/python/sglang/kernels/jit/csrc/distributed/custom_all_reduce.cuh @@ -136,7 +136,7 @@ struct MultiCastImpl { #define ALL_REDUCE_KERNEL __global__ __launch_bounds__(1024, 1) -template +template ALL_REDUCE_KERNEL void all_reduce_1shot_push_kernel(const __grid_constant__ AllReducePushParams params) { using namespace device; constexpr uint32_t kVecSize = 16 / (sizeof(T) * 2); @@ -224,25 +224,26 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR using Lamport = distributed::LamportTrait; const auto r = params.rank; const auto num_vecs = params.num_vecs; - const auto chunk_vecs = (num_vecs + kWorldSize - 1) / kWorldSize; - const auto chunk_len = [&](uint32_t c) { return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs)); }; + const auto chunk_vecs = div_ceil(num_vecs, kWorldSize); + const auto get_chunk_len = [&](uint32_t c) { // + return min(chunk_vecs, num_vecs - min(num_vecs, c * chunk_vecs)); + }; const auto num_threads = blockDim.x * gridDim.x; const auto num_warps = num_threads / kWarpThreads; const auto warp_in_block = threadIdx.x / kWarpThreads; const auto lane_id = threadIdx.x % kWarpThreads; const auto global_warp_id = blockIdx.x + gridDim.x * warp_in_block; const auto global_tid = global_warp_id * kWarpThreads + lane_id; - PDLWaitPrimary(); - const auto epoch = distributed::PushEpoch{params.ws}; - // Each warp serves one peer, starting at r + 1 so every peer's ingress is - // busy from the first wave and the slot pointer stays warp-uniform. - auto peer = r + 1 + global_warp_id % kFan; - if (peer >= kWorldSize) peer -= kWorldSize; + // peer will cover 0..kWorldSize - 1, except for local rank `r` + const auto peer = (r + 1 + global_warp_id % kFan) % kWorldSize; const auto group_warps = num_warps / kFan + (global_warp_id % kFan < num_warps % kFan ? 1 : 0); const auto group_tid = (global_warp_id / kFan) * kWarpThreads + lane_id; const auto group_stride = group_warps * kWarpThreads; - const auto peer_len = chunk_len(peer); + const auto peer_len = get_chunk_len(peer); + + PDLWaitPrimary(); + const auto epoch = distributed::PushEpoch{params.ws}; // reduce-scatter: my partial of the peer's chunk to its owner { @@ -265,17 +266,12 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR } // reduce my chunk, keep it and fan it out - const auto my_len = chunk_len(r); + const auto my_len = get_chunk_len(r); for (auto j = global_tid; j < my_len; j += num_threads) { const auto vid = r * chunk_vecs + j; vec_t vec[kWorldSize]; -#pragma unroll - for (uint32_t i = 0; i < kWorldSize; ++i) { - if (i == r) { - vec[i].load(params.input, vid); - Lamport::clear_pos_zero(vec[i].data()); - } - } + vec_t local_vec; + local_vec.load(params.input, vid); do { bool has_zero = false; #pragma unroll @@ -284,10 +280,14 @@ ALL_REDUCE_KERNEL void all_reduce_2shot_push_kernel(const __grid_constant__ AllR } #pragma unroll for (uint32_t i = 0; i < kWorldSize; ++i) { - if (i != r) has_zero |= Lamport::has_pos_zero(vec[i].data()); + has_zero |= ((i != r) & Lamport::has_pos_zero(vec[i].data())); } if (!has_zero) break; } while (true); +#pragma unroll + for (uint32_t i = 0; i < kWorldSize; ++i) { + if (i == r) vec[i] = local_vec; + } auto out_vec = reduce_vec(vec); // remap before the local store too, so every rank holds identical bits Lamport::clear_pos_zero(out_vec.data()); @@ -451,13 +451,12 @@ struct AllReduceKernel { .rank = push.rank, .ws = push.get_workspace(two_shot ? int64_t{2} * chunk_vecs * 16 : nbytes), }; - using Impl = LoadStoreImpl; const auto kernel = two_shot ? all_reduce_2shot_push_kernel - : all_reduce_1shot_push_kernel; + : all_reduce_1shot_push_kernel; // the grid is bound to the counter array and must stay constant auto block_size = choose_block_size(num_vecs); if (two_shot) { - // Every peer needs a warp, including when callers use a small grid. + // at least `kWorldSize - 1` warps in all, 1 for each peer block_size = std::max(block_size, 32u * div_ceil(kWorldSize - 1, push.num_blocks)); } LaunchKernel(push.num_blocks, block_size, stream) // diff --git a/python/sglang/srt/distributed/device_communicators/configs/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/configs/custom_all_reduce_v2.py index c434ee861b..26ffd5503d 100644 --- a/python/sglang/srt/distributed/device_communicators/configs/custom_all_reduce_v2.py +++ b/python/sglang/srt/distributed/device_communicators/configs/custom_all_reduce_v2.py @@ -31,26 +31,36 @@ class Range(NamedTuple): class Heuristic(NamedTuple): """Self-contained algo ranges for one dispatch context (graph or eager). - Four algos are tried in order of preference (fastest first): + Five algos are tried in order of preference (fastest first): 1. ``1shot_push``: nbytes <= ``one_shot_push_threshold`` - 2. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold`` - 3. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes`` + 2. ``2shot_push``: nbytes <= ``two_shot_push_threshold`` + 3. ``1shot_pull``: nbytes <= ``one_shot_pull_threshold`` + 4. ``2shot_pull`` mc: nbytes in ``mc.min_bytes..mc.max_bytes`` (only when multicast is enabled at runtime) - 4. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold`` + 5. ``2shot_pull``: nbytes <= ``two_shot_pull_threshold`` Above all of these, the caller falls back to NCCL. Setting two adjacent thresholds equal effectively disables the middle - algo; leaving ``mc`` at the default disables multicast. + algo; leaving ``mc`` / ``two_shot_push_threshold`` at the default + disables that algo. + + The field order below is the positional layout of the table rows + (``1shot_push, 1shot_pull, 2shot_pull, mc``); ``two_shot_push_threshold`` + is set by keyword, see ``_sm100_configs``. """ one_shot_push_threshold: int one_shot_pull_threshold: int two_shot_pull_threshold: int mc: Range = Range(0, 0) # default: multicast disabled in this context + two_shot_push_threshold: int = 0 # default: 2shot_push disabled - @property - def max_push_bytes(self) -> int: - return self.one_shot_push_threshold + def max_push_bytes(self, world_size: int) -> int: + """Per-slot push plane bytes this heuristic needs.""" + return max( + self.one_shot_push_threshold, + two_shot_push_slot_bytes(self.two_shot_push_threshold, world_size), + ) @property def max_pull_bytes(self) -> int: @@ -62,15 +72,32 @@ class Heuristic(NamedTuple): self.mc.max_bytes, ) - def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "Heuristic": + def clip( + self, *, max_push_bytes: int, max_pull_bytes: int, world_size: int + ) -> "Heuristic": return Heuristic( - min(self.one_shot_push_threshold, max_push_bytes), - min(self.one_shot_pull_threshold, max_pull_bytes), - min(self.two_shot_pull_threshold, max_pull_bytes), - self.mc.clip(max_pull_bytes), + one_shot_push_threshold=min(self.one_shot_push_threshold, max_push_bytes), + one_shot_pull_threshold=min(self.one_shot_pull_threshold, max_pull_bytes), + two_shot_pull_threshold=min(self.two_shot_pull_threshold, max_pull_bytes), + mc=self.mc.clip(max_pull_bytes), + two_shot_push_threshold=min( + self.two_shot_push_threshold, + two_shot_push_max_bytes(max_push_bytes, world_size), + ), ) +def two_shot_push_slot_bytes(nbytes: int, world_size: int) -> int: + """A 2shot_push slot holds a partial half and a gathered half of one + ``ceil(nbytes / world_size)`` chunk, in 16 B vectors.""" + return 32 * -(-nbytes // (16 * world_size)) + + +def two_shot_push_max_bytes(slot_bytes: int, world_size: int) -> int: + """Inverse of ``two_shot_push_slot_bytes``: the largest tensor a slot fits.""" + return world_size * 16 * (slot_bytes // 32) + + class AllReduceConfig(NamedTuple): """All tuning knobs for a single (arch, world_size). @@ -88,22 +115,19 @@ class AllReduceConfig(NamedTuple): num_pull_blocks: int num_mc_blocks: Optional[int] - @property - def max_push_bytes(self) -> int: - return max(self.graph.max_push_bytes, self.eager.max_push_bytes) + def max_push_bytes(self, world_size: int) -> int: + return max( + self.graph.max_push_bytes(world_size), + self.eager.max_push_bytes(world_size), + ) @property def max_pull_bytes(self) -> int: return max(self.graph.max_pull_bytes, self.eager.max_pull_bytes) - def clip(self, *, max_push_bytes: int, max_pull_bytes: int) -> "AllReduceConfig": + def clip(self, **kwargs) -> "AllReduceConfig": return self._replace( - graph=self.graph.clip( - max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes - ), - eager=self.eager.clip( - max_push_bytes=max_push_bytes, max_pull_bytes=max_pull_bytes - ), + graph=self.graph.clip(**kwargs), eager=self.eager.clip(**kwargs) ) @@ -112,15 +136,22 @@ def _pack_heuristic(*args) -> Heuristic: return Heuristic(*arg_list) -# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200. +# SM100 (Blackwell, B200/B300/GB200). Tuned on B200 (148 SMs); world 16 on GB200; +# the 2shot_push rows (world 4 / 8) on B300. @cache def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]: mc_blocks = {5: 64, 6: 48, 7: 48, 8: 32, 16: 32} - def config(world_size: int, *, graph: tuple, eager: tuple) -> AllReduceConfig: + def config( + world_size: int, *, graph: tuple, eager: tuple, two_shot_push: int = 0 + ) -> AllReduceConfig: return AllReduceConfig( - graph=_pack_heuristic(*graph), - eager=_pack_heuristic(*eager), + graph=_pack_heuristic(*graph)._replace( + two_shot_push_threshold=two_shot_push + ), + eager=_pack_heuristic(*eager)._replace( + two_shot_push_threshold=two_shot_push + ), num_push_blocks=num_sm, num_pull_blocks=num_sm if world_size == 2 else 96, num_mc_blocks=mc_blocks.get(world_size), @@ -139,8 +170,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]: ), 4: config( 4, - graph=(2.250 * MB, 2.250 * MB, 128.0 * MB), - eager=(3.000 * MB, 3.000 * MB, 32.00 * MB), + graph=(0.625 * MB, 0.625 * MB, 128.0 * MB), + eager=(0.625 * MB, 0.625 * MB, 32.00 * MB), + two_shot_push=8 * MB, ), 5: config( 5, @@ -159,8 +191,9 @@ def _sm100_configs(num_sm: int) -> dict[int, AllReduceConfig]: ), 8: config( 8, - graph=(0.500 * MB, 0.500 * MB, 128.0 * MB, Range(8 * MB, 128 * MB)), - eager=(0.750 * MB, 0.750 * MB, 128.0 * MB, Range(0, 128 * MB)), + graph=(256 * KB, 256 * KB, 128.0 * MB, Range(8 * MB, 128 * MB)), + eager=(256 * KB, 256 * KB, 128.0 * MB, Range(0, 128 * MB)), + two_shot_push=8 * MB, ), 16: config( 16, diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py index 15b0bbadae..9109f1bc89 100644 --- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py @@ -145,7 +145,7 @@ class CustomAllReduceV2: if max_pull_size is None: max_pull_size = min(base_config.max_pull_bytes, max_size) if max_push_size is None: - max_push_size = min(base_config.max_push_bytes, max_size) + max_push_size = min(base_config.max_push_bytes(self.world_size), max_size) if _FORCE_PULL_SIZE_KB is not None: max_pull_size = int(_FORCE_PULL_SIZE_KB) * 1024 if _FORCE_PUSH_SIZE_KB is not None: @@ -185,7 +185,9 @@ class CustomAllReduceV2: if max_push_blocks is not None: num_push_blocks = max(max_push_blocks, 1) self.config = base_config.clip( - max_push_bytes=self.max_push_size, max_pull_bytes=self.max_pull_size + max_push_bytes=self.max_push_size, + max_pull_bytes=self.max_pull_size, + world_size=self.world_size, )._replace(num_pull_blocks=num_pull_blocks, num_push_blocks=num_push_blocks) self.override_algo: Optional[AllReduceAlgo] = None # On a multi-node (MNNVL) group the symm-mem workspace plane works @@ -338,6 +340,8 @@ class CustomAllReduceV2: can_use_multicast = self.config.num_mc_blocks is not None if nbytes <= heuristic.one_shot_push_threshold: return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PUSH) + if nbytes <= heuristic.two_shot_push_threshold: + return AllReduceConfig(AllReduceAlgo.TWO_SHOT_PUSH) if nbytes <= heuristic.one_shot_pull_threshold: return AllReduceConfig(AllReduceAlgo.ONE_SHOT_PULL, use_graph=can_use_graph) if can_use_multicast and heuristic.mc.contains(nbytes): diff --git a/test/registered/kernels/ops/communication/test_custom_all_reduce.py b/test/registered/kernels/ops/communication/test_custom_all_reduce.py index 2e3787a519..941fdf444b 100644 --- a/test/registered/kernels/ops/communication/test_custom_all_reduce.py +++ b/test/registered/kernels/ops/communication/test_custom_all_reduce.py @@ -63,12 +63,14 @@ TEST_SIZES = [ 32 * 1024, 256 * 1024, 2 * 1024 * 1024, + 3 * 1024 * 1024 + 24, # chunks not a multiple of world_size 4 * 1024 * 1024, ] TEST_DTYPES = [torch.float16, torch.bfloat16, torch.float32] TEST_ALGOS = [ AllReduceAlgo.ONE_SHOT_PULL, AllReduceAlgo.ONE_SHOT_PUSH, + AllReduceAlgo.TWO_SHOT_PUSH, AllReduceAlgo.TWO_SHOT_PULL, ] USE_GRAPH_OPTIONS = [False, True]