[SM120] Add optional FlashInfer PCIe-IPC all-reduce for switch-free hosts (#34528)

This commit is contained in:
eeecho
2026-09-28 23:08:44 -07:00
committed by GitHub
parent 2ff52e3ceb
commit c7be3e935b
6 changed files with 1030 additions and 0 deletions
@@ -0,0 +1,426 @@
# SPDX-License-Identifier: Apache-2.0
"""FlashInfer PCIe-IPC all-reduce for switch-free intra-node machines.
FlashInfer's ``pcie_ipc_comm`` targets hosts where every peer transfer crosses the
CPU root complex -- no NVLink, no multicast. The existing custom all-reduce
backends assume one of those fabrics, so on such a host every per-layer reduction
falls back to NCCL, which leaves a large margin on the table at decode sizes.
Why the kernel wins here
------------------------
Naive all-to-all peer writes collapse on this fabric, and the collapse worsens
with the number of concurrently writing blocks. FlashInfer's kernels instead
*stage* their pushes so each rank has exactly one outbound and one inbound stream
at any instant, and the 8-rank path keeps a 4+4 island decomposition so the
scarce cross-socket link carries the minimum. Measured on one 8x SM120 host
(GPUs 0-3 and 4-7 on separate NUMA nodes), against NCCL, hidden 6144 bf16:
========= =========== ========== =========
Message FlashInfer NCCL Speed-up
========= =========== ========== =========
12 KiB 5.3 us 205.1 us 38.8x
96 KiB 15.3 us 584.9 us 38.2x
384 KiB 34.6 us 1169.3 us 33.8x
100 MiB 8.5 ms 50.4 ms 5.9x
========= =========== ========== =========
:meth:`PcieIpcAllReduceWorkspace.supports` reports whether the kernels can run
a shape, not whether they beat NCCL on it, so the workspace bound below is what
keeps the reductions this backend should not serve on NCCL.
Workspace sizing
----------------
The workspace is sized for the widest decode reduction the runner will issue,
``max_bs * draft_tokens * hidden``. The bound is a row count, not a phase
split: a prefill chunk narrower than it is admitted too. What the bound rules
out is the full-width prefill chunk, which is orders of magnitude larger, and
there NCCL's ring is the better algorithm. Measured at 8 ranks, TP8, 8k
context, against NCCL as the baseline:
=================== ========== ========== ===============
Workspace TTFT TPOT Output tok/s
=================== ========== ========== ===============
NCCL only 1910 ms 21.14 ms 35.02
prefill-sized 3176 ms 13.64 ms 38.43
decode-sized 1849 ms 13.62 ms 48.05
=================== ========== ========== ===============
Handing prefill to these kernels costs 66% on TTFT and buys nothing on TPOT.
The decode-sized workspace keeps the whole TPOT win, stays level with NCCL on
TTFT, and is the only one of the three that is not worse than NCCL anywhere we
measured. It is also ~250x smaller, which matters at long context: at 128k with
4 concurrent requests the prefill-sized workspace regressed TPOT by 45%, and
that regression disappears when it is sized for decode.
The bound comes from the resolved decode config, but the hidden size is not
known when the group is built, so the workspace is created on the first
eligible tensor, whose trailing dimension is exactly that hidden size. Ranks
run the same sequence of reductions, so they all reach that first call with the
same shape and build the same workspace. ``SGLANG_PCIE_IPC_MAX_NUMEL`` overrides
the derivation for deployments that want to hand larger reductions to the
kernels anyway.
"""
import functools
import logging
from typing import Any, Optional
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from sglang.srt.environ import envs
logger = logging.getLogger(__name__)
# The kernels build IPC channels for these world sizes only.
_SUPPORTED_WORLD_SIZES = (2, 4, 8)
# Fallback decode width when the server args are absent (unit tests, embedded
# use). Covers the batch sizes we serve; anything larger stays on NCCL.
_FALLBACK_DECODE_WIDTH = 64
#: attention_tp / moe_tp / pdmux_prefill_tp carry no pynccl or custom
#: all-reduce communicator, so a reduction dispatched to them asserts.
_ELIGIBLE_GROUP_NAMES = frozenset({"tp"})
@functools.lru_cache(maxsize=None)
def _warn_deterministic_wins() -> None:
logger.warning(
"Deterministic inference is on; leaving PCIe-IPC all-reduce off. Its "
"kernels pick a tactic per shape, so the reduction order would follow "
"the batch shape."
)
def eligible_group(
*, group_name: Optional[str], world_size: int, deterministic: bool
) -> bool:
"""Whether ``GroupCoordinator`` should build this backend for a group."""
if not (
envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.get()
and world_size > 1
and group_name in _ELIGIBLE_GROUP_NAMES
):
return False
# Tactic is chosen per shape, so the reduction order would follow the
# batch shape; disabling tuning alone does not help, the seed policy is
# per-shape too.
if deterministic:
_warn_deterministic_wins()
return False
return True
@functools.lru_cache(maxsize=None)
def _warn_untuned_dtype(dtype: torch.dtype) -> None:
"""Once per dtype: a per-layer warning would be issued thousands of times."""
logger.warning(
"FlashInfer PCIe-IPC all-reduce serves bf16 only; %s reductions stay on NCCL.",
dtype,
)
def _autotune_disabled() -> bool:
"""Whether the operator asked for no FlashInfer autotuning."""
from sglang.srt.runtime_context import get_exec
try:
return bool(get_exec().kernel.disable_flashinfer_autotune)
except ValueError:
return False
def _decode_width() -> Optional[int]:
"""Rows in the widest decode reduction, or None when the config is unavailable.
Decode runs inside a captured graph, so the largest captured batch bounds
the reduction. ``max_bs`` counts requests, not rows: a speculative forward
verifies several draft tokens per sequence, so the row count is
``max_bs * draft tokens``, as in
``runtime_context.cutedsl_moe_max_num_tokens``.
"""
from sglang.srt.runtime_context import get_exec, get_spec
try:
cg_config = get_exec().graph.cuda_graph_config
spec = get_spec()
except ValueError:
# Embedded and unit-test use have no runtime context; the caller then
# sizes from a fixed width that ignores the server args, so say so.
logger.warning(
"FlashInfer PCIe-IPC has no runtime context to read the decode "
"width from; falling back to a fixed bound."
)
return None
if cg_config is None or not cg_config.decode.max_bs:
return None
rows_per_req = (
(spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1
)
return cg_config.decode.max_bs * rows_per_req
def _tune_cache_path(world_size: int) -> Optional[str]:
"""Where FlashInfer should persist the measured tactics.
Named explicitly rather than left to default, because the tuning run has to
write the file for the *next* server to reuse it; a workspace built without
a cache path measures every start.
"""
try:
from flashinfer.comm.pcie_ipc_tuning import default_cache_path
return str(default_cache_path(world_size))
except (AttributeError, ImportError):
return None
def _tune_batches(max_rows: int) -> tuple[int, ...]:
"""Row counts to profile, bounded by what the workspace can actually take.
The autotuner measures one tactic per shape, so profiling rows the workspace
would reject is wasted startup time. Powers of two cover the captured decode
batches; ``max_rows`` is appended because it is the widest reduction served
and is not always a power of two.
"""
rows = [r for r in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) if r <= max_rows]
if max_rows >= 1 and max_rows not in rows:
rows.append(max_rows)
return tuple(rows)
class PcieIpcCommunicator:
"""Adapter between ``GroupCoordinator`` and FlashInfer's PCIe-IPC all-reduce."""
def __init__(
self,
group: ProcessGroup,
device: torch.device | int,
cpu_group: Optional[ProcessGroup] = None,
):
self.disabled = True
self.max_numel = 0
self._workspace: Optional[Any] = None
self._bound_stream: Optional[torch.cuda.Stream] = None
self._cpu_group = cpu_group
self._max_rows: Optional[int] = None
world_size = dist.get_world_size(group=group)
if world_size not in _SUPPORTED_WORLD_SIZES:
logger.debug(
"FlashInfer PCIe-IPC all-reduce skipped: unsupported world size %d",
world_size,
)
return
if isinstance(device, int):
device = torch.device(f"cuda:{device}")
if device.type != "cuda":
return
try:
from flashinfer.comm import PcieIpcAllReduceWorkspace
except ImportError:
logger.warning(
"SGLANG_ENABLE_PCIE_IPC_ALLREDUCE is set but this FlashInfer build has "
"no pcie_ipc_comm; falling back to NCCL all-reduce."
)
return
# The workspace is built on the first eligible tensor: sizing it needs the
# hidden size, which the group does not know yet.
self._group = group
self._device = device
self._world_size = world_size
self._workspace_cls = PcieIpcAllReduceWorkspace
self._build_failed = False
self.disabled = False
logger.info(
"FlashInfer PCIe-IPC all-reduce enabled (world=%d, workspace sized on "
"first reduction)",
world_size,
)
def _ensure_workspace(self, inp: torch.Tensor) -> bool:
"""Build the workspace for the widest decode, using ``inp`` for the hidden size.
Every rank issues the same reductions in the same order, so they all arrive
here with the same shape and agree on ``max_numel`` without extra exchange.
"""
if self._workspace is not None:
return True
if self._build_failed:
return False
hidden = inp.shape[-1]
override = envs.SGLANG_PCIE_IPC_MAX_NUMEL.get()
if override:
max_numel = override
else:
# The runner's width is already clamped to the request pool;
# _decode_width() can only re-derive it.
width = self._max_rows if self._max_rows else _decode_width()
if width is None:
# Admits a different set of batches than the server issues;
# every reduction outside it silently uses NCCL.
logger.warning(
"FlashInfer PCIe-IPC is sizing its workspace from the fixed "
"fallback width %d, not from this server's decode config; "
"set SGLANG_PCIE_IPC_MAX_NUMEL to override.",
_FALLBACK_DECODE_WIDTH,
)
width = _FALLBACK_DECODE_WIDTH
max_numel = width * hidden
try:
self._workspace = self._workspace_cls(
group=self._group,
max_numel=max_numel,
dtype=torch.bfloat16,
tune_batches=_tune_batches(max_numel // hidden),
tune_cache=_tune_cache_path(self._world_size),
)
except Exception as e:
logger.warning(
"FlashInfer PCIe-IPC workspace (max_numel=%d) failed: %s; "
"reductions stay on NCCL",
max_numel,
e,
)
self._build_failed = True
self.disabled = True
return False
self.max_numel = max_numel
logger.info(
"FlashInfer PCIe-IPC workspace built (world=%d, max_numel=%d, hidden=%d)",
self._world_size,
max_numel,
hidden,
)
self._tune(hidden)
return True
def prepare(self, *, hidden: int, max_rows: Optional[int] = None) -> None:
"""Build and measure ahead of the first reduction.
Tuning runs here rather than on the first reduction, which may land
inside graph capture or another autotune pass; ``_tune`` declines in
both, leaving the kernels on FlashInfer's seed policy.
``max_rows`` is the widest decode reduction the runner will issue.
"""
if self.disabled or self._workspace is not None:
return
self._max_rows = max_rows
probe = torch.empty((1, hidden), dtype=torch.bfloat16, device=self._device)
self._ensure_workspace(probe)
def _tune(self, hidden: int) -> None:
"""Measure the launch tactics for this workspace's shapes.
Without this the kernels run FlashInfer's seed policy, and ``supports``
answers from that policy rather than from measurements on this host.
Results persist to FlashInfer's cache, so only the first server pays.
"""
if _autotune_disabled():
logger.info(
"FlashInfer PCIe-IPC all-reduce: --disable-flashinfer-autotune is "
"set; keeping FlashInfer's seed policy."
)
return
from flashinfer.autotuner import AutoTuner
if AutoTuner.get().is_tuning_mode:
# Reached through the lazy build, so tuning would start at
# whatever point the first reduction happens; keep it to prepare().
logger.warning(
"FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
"another autotune context; skipping autotune and keeping the seed "
"policy. Call PcieIpcCommunicator.prepare() from warmup to tune."
)
return
if self._cpu_group is None:
logger.warning(
"FlashInfer PCIe-IPC all-reduce has no host group to autotune on; "
"running FlashInfer's seed policy instead of measured tactics."
)
return
if torch.cuda.is_current_stream_capturing():
logger.warning(
"FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
"a graph capture; skipping autotune and keeping the seed policy."
)
return
try:
tuned = self._workspace.tune(
[hidden], dtype=torch.bfloat16, tune_group=self._cpu_group
)
except Exception as e:
logger.warning(
"FlashInfer PCIe-IPC autotune failed (%s); keeping the seed policy.",
e,
)
return
# Report what tune() covered, not that it returned: it declines shapes
# silently, and a log line that only proves no exception was raised is
# how an autotune that measured nothing reads as a success.
if not tuned:
logger.warning(
"FlashInfer PCIe-IPC autotune covered no shapes for hidden=%d "
"(requested rows %s); the kernels keep the seed policy.",
hidden,
_tune_batches(self.max_numel // hidden),
)
return
logger.info(
"FlashInfer PCIe-IPC all-reduce autotuned %d shape(s) (hidden=%d, rows=%s)",
len(tuned),
hidden,
_tune_batches(self.max_numel // hidden),
)
def should_pcie_ipc_ar(self, inp: torch.Tensor) -> bool:
"""Whether this reduction should go to the kernels rather than NCCL.
``supports`` answers whether the kernels can run the tensor, not
whether they beat NCCL on it, so the size bound above is what keeps
prefill-sized reductions on NCCL.
"""
if self.disabled or not inp.is_contiguous() or inp.dim() < 2:
return False
# FlashInfer keys tuning results by dtype, and only bf16 is tuned here.
if inp.dtype is not torch.bfloat16:
_warn_untuned_dtype(inp.dtype)
return False
if not self._ensure_workspace(inp):
return False
if inp.numel() > self.max_numel:
return False
return self._workspace.supports(inp)
def pcie_ipc_all_reduce(self, inp: torch.Tensor) -> Optional[torch.Tensor]:
"""All-reduce ``inp``, rebinding the workspace when the stream changes.
One workspace serves one stream: its epoch and arrival counters assume the
calls sharing it are totally ordered. SGLang switches streams at phase
boundaries -- graph capture, then replay -- so the binding is moved with
the caller rather than kept on whichever stream got there first. This is
safe because those phases do not overlap; it would not be safe if two
streams issued reductions concurrently on the same group.
"""
stream = torch.cuda.current_stream()
if self._bound_stream is None or stream != self._bound_stream:
self._workspace.rebind_stream()
self._bound_stream = stream
return self._workspace.all_reduce(inp)
def destroy(self) -> None:
if self._workspace is not None:
self._workspace.destroy()
self._workspace = None
self.disabled = True
@@ -485,6 +485,28 @@ class GroupCoordinator:
self.ca_comm: Optional[Any] = None
self.qr_comm: Optional[QuickAllReduce] = None
self.pcie_ipc_comm: Optional[Any] = None
from sglang.srt.distributed.device_communicators.pcie_ipc_ar import (
PcieIpcCommunicator,
eligible_group,
)
if eligible_group(
group_name=group_name,
world_size=self.world_size,
deterministic=self._deterministic_collectives_enabled(),
):
try:
# The IPC handshake needs the CUDA (NCCL) group, not the CPU one.
# Autotuning is the other way round: it rendezvouses on the host.
self.pcie_ipc_comm = PcieIpcCommunicator(
group=self.device_group,
device=self.device,
cpu_group=self.cpu_group,
)
except Exception as e:
logger.warning(f"Setup FlashInfer PCIe-IPC allreduce failed with {e}.")
if use_custom_allreduce and self.world_size > 1:
# Initialize a custom fast all-reduce implementation.
try:
@@ -944,6 +966,16 @@ class GroupCoordinator:
and self.ca_comm.should_custom_ar(input_)
):
return "ca"
# After ``ca``: the PCIe-IPC kernels are for hosts where no fabric-specific
# backend applies. They do not probe for NVLink, so on a host that has it
# this ordering is what keeps the faster backend in front of them.
if (
self.pcie_ipc_comm is not None
and not self.pcie_ipc_comm.disabled
and not should_use_pymscclpp_allreduce
and self.pcie_ipc_comm.should_pcie_ipc_ar(input_)
):
return "pcie_ipc"
if (
self.qr_comm is not None
and not self.qr_comm.disabled
@@ -1025,6 +1057,8 @@ class GroupCoordinator:
elif outplace_all_reduce_method == "pymscclpp":
assert not pymscclpp_comm.disabled
out = pymscclpp_comm.all_reduce(input_)
elif outplace_all_reduce_method == "pcie_ipc":
return self.pcie_ipc_comm.pcie_ipc_all_reduce(input_)
elif outplace_all_reduce_method == "pynccl":
with pynccl_comm.change_state(enable=True):
out = pynccl_comm.outplace_all_reduce(input_)
@@ -2059,6 +2093,11 @@ class GroupCoordinator:
return tensor
def destroy(self):
# Must precede destroy_process_group(): FlashInfer's workspace
# teardown collectives on the group it was built with.
if self.pcie_ipc_comm is not None:
self.pcie_ipc_comm.destroy()
self.pcie_ipc_comm = None
if self.device_group is not None:
torch.distributed.destroy_process_group(self.device_group)
self.device_group = None
+3
View File
@@ -1350,6 +1350,9 @@ class Envs:
SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB = EnvInt(16 * 1024)
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PULL_SIZE_KB = EnvInt(None)
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PUSH_SIZE_KB = EnvInt(None)
# See docs/references/environment_variables.
SGLANG_ENABLE_PCIE_IPC_ALLREDUCE = EnvBool(False)
SGLANG_PCIE_IPC_MAX_NUMEL = EnvInt(0)
# ===================================================================
# RoPE cache
@@ -253,6 +253,7 @@ class BaseRunner(ABC):
self._pre_initialize_flashinfer_allreduce_workspace()
self._pre_initialize_fi_a2a_workspace()
self._pre_initialize_pcie_ipc_workspace()
# Model-owned communication resources may depend on the resolved
# request pool and must be compiled/allocated before graph capture.
@@ -333,6 +334,44 @@ class BaseRunner(ABC):
init_fi_a2a_workspace(get_parallel().dcp_group)
def _pre_initialize_pcie_ipc_workspace(self):
"""Build the PCIe-IPC all-reduce workspace before graph capture.
Runs for every model, not only the ones that autotune: left to the first
reduction, the build lands inside another autotune context and tuning
declines there.
"""
pcie_ipc_comm = get_parallel().tp_group.pcie_ipc_comm
if pcie_ipc_comm is None:
return
mr = self.model_runner
pcie_ipc_comm.prepare(
hidden=mr.model_config.hidden_size,
max_rows=self._widest_decode_rows(),
)
def _widest_decode_rows(self) -> Optional[int]:
"""Rows in the widest decode reduction, or None when no graph is captured.
Same derivation the decode runner uses for its buffers, so the workspace
covers exactly the batches that will be issued: get_batch_sizes_to_capture
already applies the attention-tp alignment and the req_to_token_pool clamp,
which a second derivation from cuda_graph_config would miss.
"""
if get_exec().graph.cuda_graph_config is None:
return None
from sglang.srt.model_executor.runner.base_cuda_graph_runner import (
get_batch_sizes_to_capture,
)
mr = self.model_runner
tokens_per_req = mr.decode_num_tokens_per_req()
capture_bs, _ = get_batch_sizes_to_capture(mr, tokens_per_req)
if not capture_bs:
return None
return max(capture_bs) * tokens_per_req
def _flashinfer_autotune(self, *, buffers, batch_size):
"""Run flashinfer autotune.