[Perf] distributed: cache the eager launch of custom all-reduce v2 per size (#44642)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
b234b5c32e
commit
f53bd436c1
@@ -32,6 +32,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.kernels.kernel_api_logging import _KERNEL_API_LOG_LEVEL
|
||||
from sglang.kernels.ops.communication.all_reduce import (
|
||||
AllReduceAlgo,
|
||||
Communicator,
|
||||
@@ -39,6 +40,7 @@ from sglang.kernels.ops.communication.all_reduce import (
|
||||
PullPlane,
|
||||
PushPlane,
|
||||
custom_all_reduce,
|
||||
get_all_reduce_module,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import in_the_same_node_as
|
||||
from sglang.srt.environ import envs
|
||||
@@ -190,6 +192,10 @@ class CustomAllReduceV2:
|
||||
world_size=self.world_size,
|
||||
)._replace(num_pull_blocks=num_pull_blocks, num_push_blocks=num_push_blocks)
|
||||
self.override_algo: Optional[AllReduceAlgo] = None
|
||||
# (nbytes, dtype) -> (module, algo name, multicast) or None, for
|
||||
# eager calls under `_eager_plans_config`; rebuilt when config changes.
|
||||
self._eager_plans: dict = {}
|
||||
self._eager_plans_config = None
|
||||
# On a multi-node (MNNVL) group the symm-mem workspace plane works
|
||||
# across nodes, but graph zero-copy input registration (cudaIpc /
|
||||
# node-local VMM remap) does not — force eager pull inside graphs.
|
||||
@@ -362,14 +368,44 @@ class CustomAllReduceV2:
|
||||
return False
|
||||
if self.override_algo is not None:
|
||||
return inp_size <= self.max_size
|
||||
if not self._graph_mode_allowed:
|
||||
return self._eager_plan(inp_size, inp.dtype) is not None
|
||||
return self._pick_config(inp_size, self._can_use_graph()) is not None
|
||||
|
||||
def _eager_plan(self, nbytes: int, dtype: torch.dtype):
|
||||
"""The launch an eager all-reduce of this size takes, resolved once."""
|
||||
if self._eager_plans_config is not self.config:
|
||||
self._eager_plans, self._eager_plans_config = {}, self.config
|
||||
key = (nbytes, dtype)
|
||||
if key not in self._eager_plans:
|
||||
config = self._pick_config(nbytes, False)
|
||||
self._eager_plans[key] = (
|
||||
None
|
||||
if config is None
|
||||
else (
|
||||
get_all_reduce_module(dtype, self.world_size),
|
||||
config.algo.algo_name,
|
||||
config.use_multicast,
|
||||
)
|
||||
)
|
||||
return self._eager_plans[key]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# All-reduce
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor:
|
||||
nbytes = input.numel() * input.element_size()
|
||||
if (
|
||||
self.override_algo is None
|
||||
and not self._graph_mode_allowed
|
||||
and not _KERNEL_API_LOG_LEVEL
|
||||
):
|
||||
plan = self._eager_plan(nbytes, input.dtype)
|
||||
assert plan is not None, f"No config for {nbytes = }"
|
||||
module, algo_name, use_multicast = plan
|
||||
out = module.all_reduce(self.obj, input, algo_name, None, use_multicast)
|
||||
return out if isinstance(out, torch.Tensor) else torch.from_dlpack(out)
|
||||
if self.override_algo is not None:
|
||||
# TODO: enhance this override pattern
|
||||
algo = self.override_algo
|
||||
|
||||
Reference in New Issue
Block a user