diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py index 9109f1bc89..8f68572ed4 100644 --- a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_v2.py @@ -32,6 +32,7 @@ import torch import torch.distributed as dist from torch.distributed import ProcessGroup +from sglang.kernels.kernel_api_logging import _KERNEL_API_LOG_LEVEL from sglang.kernels.ops.communication.all_reduce import ( AllReduceAlgo, Communicator, @@ -39,6 +40,7 @@ from sglang.kernels.ops.communication.all_reduce import ( PullPlane, PushPlane, custom_all_reduce, + get_all_reduce_module, ) from sglang.srt.distributed.parallel_state import in_the_same_node_as from sglang.srt.environ import envs @@ -190,6 +192,10 @@ class CustomAllReduceV2: world_size=self.world_size, )._replace(num_pull_blocks=num_pull_blocks, num_push_blocks=num_push_blocks) self.override_algo: Optional[AllReduceAlgo] = None + # (nbytes, dtype) -> (module, algo name, multicast) or None, for + # eager calls under `_eager_plans_config`; rebuilt when config changes. + self._eager_plans: dict = {} + self._eager_plans_config = None # On a multi-node (MNNVL) group the symm-mem workspace plane works # across nodes, but graph zero-copy input registration (cudaIpc / # node-local VMM remap) does not — force eager pull inside graphs. @@ -362,14 +368,44 @@ class CustomAllReduceV2: return False if self.override_algo is not None: return inp_size <= self.max_size + if not self._graph_mode_allowed: + return self._eager_plan(inp_size, inp.dtype) is not None return self._pick_config(inp_size, self._can_use_graph()) is not None + def _eager_plan(self, nbytes: int, dtype: torch.dtype): + """The launch an eager all-reduce of this size takes, resolved once.""" + if self._eager_plans_config is not self.config: + self._eager_plans, self._eager_plans_config = {}, self.config + key = (nbytes, dtype) + if key not in self._eager_plans: + config = self._pick_config(nbytes, False) + self._eager_plans[key] = ( + None + if config is None + else ( + get_all_reduce_module(dtype, self.world_size), + config.algo.algo_name, + config.use_multicast, + ) + ) + return self._eager_plans[key] + # ------------------------------------------------------------------ # All-reduce # ------------------------------------------------------------------ def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor: nbytes = input.numel() * input.element_size() + if ( + self.override_algo is None + and not self._graph_mode_allowed + and not _KERNEL_API_LOG_LEVEL + ): + plan = self._eager_plan(nbytes, input.dtype) + assert plan is not None, f"No config for {nbytes = }" + module, algo_name, use_multicast = plan + out = module.all_reduce(self.obj, input, algo_name, None, use_multicast) + return out if isinstance(out, torch.Tensor) else torch.from_dlpack(out) if self.override_algo is not None: # TODO: enhance this override pattern algo = self.override_algo diff --git a/test/registered/kernels/ops/communication/test_custom_all_reduce.py b/test/registered/kernels/ops/communication/test_custom_all_reduce.py index 941fdf444b..4c971bf332 100644 --- a/test/registered/kernels/ops/communication/test_custom_all_reduce.py +++ b/test/registered/kernels/ops/communication/test_custom_all_reduce.py @@ -234,6 +234,64 @@ def test_custom_all_reduce( torch.testing.assert_close(out_ref, out_jit, atol=0, rtol=0) +def _eager_all_reduce(comm: CustomAllReduceV2, x: torch.Tensor) -> torch.Tensor: + outs = [] + for row in x.clone(): + outs.append(comm.custom_all_reduce(row)) + torch.cuda.synchronize() + return torch.stack(outs) + + +@pytest.mark.parametrize("dtype", TEST_DTYPES) +@pytest.mark.parametrize("size", TEST_SIZES) +@torch.inference_mode() +def test_custom_all_reduce_tuned_eager(size: int, dtype: torch.dtype) -> None: + """Without an override, eager calls launch the tuned algorithm (cached per size).""" + nccl_group = _init_nccl_group_once() + comm = _init_comm_once() + comm.override_algo = None + device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}") + inp = torch.randint(0, 16, (TEST_LAYERS, size), dtype=dtype, device=device) + nbytes = inp[0].numel() * inp.element_size() + tuned = comm._pick_config(nbytes, False) is not None + assert comm.should_custom_ar(inp[0]) == tuned + if not tuned: + return + out_ref = inp.clone() + dist.all_reduce(out_ref, group=nccl_group) + for _ in range(2): # the second pass runs from the cached launch + torch.testing.assert_close( + out_ref, _eager_all_reduce(comm, inp), atol=0, rtol=0 + ) + + +@torch.inference_mode() +def test_custom_all_reduce_eager_plan_follows_config() -> None: + """The cached eager launch is rebuilt when the config object is replaced.""" + nccl_group = _init_nccl_group_once() + comm = _init_comm_once() + comm.override_algo = None + device = torch.device(f"cuda:{int(os.environ['LOCAL_RANK'])}") + inp = torch.randint(0, 16, (TEST_LAYERS, 1024), dtype=torch.bfloat16, device=device) + assert comm.should_custom_ar(inp[0]) # cached under the tuned config + tuned = comm.config + try: + eager = tuned.eager._replace( + one_shot_push_threshold=0, + one_shot_pull_threshold=0, + two_shot_pull_threshold=0, + mc=tuned.eager.mc._replace(min_bytes=0, max_bytes=0), + ) + comm.config = tuned._replace(eager=eager) + assert not comm.should_custom_ar(inp[0]) + finally: + comm.config = tuned + assert comm.should_custom_ar(inp[0]) + out_ref = inp.clone() + dist.all_reduce(out_ref, group=nccl_group) + torch.testing.assert_close(out_ref, _eager_all_reduce(comm, inp), atol=0, rtol=0) + + if __name__ == "__main__": # Only sweep the common world sizes (2, 4, 8, 16) by default: testing every # count in 2..16 serially overruns the per-file CI time budget, and numbers