diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 7f530651b..e9e41b988 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -906,6 +906,10 @@ class Envs: # Set to 1: force enable (even without --enable-deterministic-inference) # Set to 0: force disable (use default Aiter AR even with --enable-deterministic-inference) SGLANG_USE_1STAGE_ALLREDUCE = EnvBool(False) + # NCCL channel count pinned on CUDA so the all-reduce reduces a token the + # same way whatever else shares its batch. Raise it to buy back bandwidth + # on links that can drive more channels. + SGLANG_DETERMINISTIC_NCCL_NCHANNELS = EnvInt(8) SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 = EnvBool(True) # MiniMax-M3 on ROCm force-disables custom all-reduce in its model override # (arg_groups/overrides.py) when aiter all-reduce fusion is off. Set this to diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 5db8727a8..536773628 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -7972,8 +7972,14 @@ class ServerArgs: # symmetric-memory path only below a byte threshold, so # which reduce runs would follow the token count. self.enable_torch_symm_mem = False + # Each channel carries a differently shaped tree and the + # channel count is picked from the message size, so a + # token's reduction order would follow the token count. + nchannels = str(envs.SGLANG_DETERMINISTIC_NCCL_NCHANNELS.get()) + os.environ["NCCL_MIN_NCHANNELS"] = nchannels + os.environ["NCCL_MAX_NCHANNELS"] = nchannels logger.warning( - "NCCL_ALGO is set to 'allreduce:tree', and custom and symmetric-memory all reduce are disabled for deterministic inference when TP size > 1." + "NCCL_ALGO is set to 'allreduce:tree', the NCCL channel count is pinned, and custom and symmetric-memory all reduce are disabled for deterministic inference when TP size > 1." ) def _handle_unified_memory_pool(self):