diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index b930c847f0..24ca0f0861 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -1705,6 +1705,16 @@ SGLang supports various environment variables that can be used to configure its
Use the 1-stage all-reduce kernel on AMD (deterministic, fixed accumulation order). If unset, it is auto-enabled when deterministic inference is on. |
false |
+
+ SGLANG_ENABLE_PCIE_IPC_ALLREDUCE |
+ Use FlashInfer's PCIe-IPC all-reduce on switch-free intra-node hosts (no NVLink, no multicast), where the other custom all-reduce backends do not apply. Serves bf16 decode reductions that fit the workspace bound; everything else stays on NCCL. Left off under deterministic inference. |
+ false |
+
+
+ SGLANG_PCIE_IPC_MAX_NUMEL |
+ Elements the PCIe-IPC workspace is sized for; it cannot grow after construction. 0 derives it from the widest decode batch the runner captures for. Raising it hands larger reductions to the kernels, at roughly 2 * world_size * max_numel * itemsize bytes per rank. |
+ 0 |
+
SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE |
FlashInfer prefill split-tile size for deterministic attention. |
diff --git a/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py b/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py
new file mode 100644
index 0000000000..6a8b36db69
--- /dev/null
+++ b/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py
@@ -0,0 +1,426 @@
+# SPDX-License-Identifier: Apache-2.0
+"""FlashInfer PCIe-IPC all-reduce for switch-free intra-node machines.
+
+FlashInfer's ``pcie_ipc_comm`` targets hosts where every peer transfer crosses the
+CPU root complex -- no NVLink, no multicast. The existing custom all-reduce
+backends assume one of those fabrics, so on such a host every per-layer reduction
+falls back to NCCL, which leaves a large margin on the table at decode sizes.
+
+Why the kernel wins here
+------------------------
+Naive all-to-all peer writes collapse on this fabric, and the collapse worsens
+with the number of concurrently writing blocks. FlashInfer's kernels instead
+*stage* their pushes so each rank has exactly one outbound and one inbound stream
+at any instant, and the 8-rank path keeps a 4+4 island decomposition so the
+scarce cross-socket link carries the minimum. Measured on one 8x SM120 host
+(GPUs 0-3 and 4-7 on separate NUMA nodes), against NCCL, hidden 6144 bf16:
+
+========= =========== ========== =========
+Message FlashInfer NCCL Speed-up
+========= =========== ========== =========
+12 KiB 5.3 us 205.1 us 38.8x
+96 KiB 15.3 us 584.9 us 38.2x
+384 KiB 34.6 us 1169.3 us 33.8x
+100 MiB 8.5 ms 50.4 ms 5.9x
+========= =========== ========== =========
+
+:meth:`PcieIpcAllReduceWorkspace.supports` reports whether the kernels can run
+a shape, not whether they beat NCCL on it, so the workspace bound below is what
+keeps the reductions this backend should not serve on NCCL.
+
+Workspace sizing
+----------------
+The workspace is sized for the widest decode reduction the runner will issue,
+``max_bs * draft_tokens * hidden``. The bound is a row count, not a phase
+split: a prefill chunk narrower than it is admitted too. What the bound rules
+out is the full-width prefill chunk, which is orders of magnitude larger, and
+there NCCL's ring is the better algorithm. Measured at 8 ranks, TP8, 8k
+context, against NCCL as the baseline:
+
+=================== ========== ========== ===============
+Workspace TTFT TPOT Output tok/s
+=================== ========== ========== ===============
+NCCL only 1910 ms 21.14 ms 35.02
+prefill-sized 3176 ms 13.64 ms 38.43
+decode-sized 1849 ms 13.62 ms 48.05
+=================== ========== ========== ===============
+
+Handing prefill to these kernels costs 66% on TTFT and buys nothing on TPOT.
+The decode-sized workspace keeps the whole TPOT win, stays level with NCCL on
+TTFT, and is the only one of the three that is not worse than NCCL anywhere we
+measured. It is also ~250x smaller, which matters at long context: at 128k with
+4 concurrent requests the prefill-sized workspace regressed TPOT by 45%, and
+that regression disappears when it is sized for decode.
+
+The bound comes from the resolved decode config, but the hidden size is not
+known when the group is built, so the workspace is created on the first
+eligible tensor, whose trailing dimension is exactly that hidden size. Ranks
+run the same sequence of reductions, so they all reach that first call with the
+same shape and build the same workspace. ``SGLANG_PCIE_IPC_MAX_NUMEL`` overrides
+the derivation for deployments that want to hand larger reductions to the
+kernels anyway.
+"""
+
+import functools
+import logging
+from typing import Any, Optional
+
+import torch
+import torch.distributed as dist
+from torch.distributed import ProcessGroup
+
+from sglang.srt.environ import envs
+
+logger = logging.getLogger(__name__)
+
+# The kernels build IPC channels for these world sizes only.
+_SUPPORTED_WORLD_SIZES = (2, 4, 8)
+
+# Fallback decode width when the server args are absent (unit tests, embedded
+# use). Covers the batch sizes we serve; anything larger stays on NCCL.
+_FALLBACK_DECODE_WIDTH = 64
+
+
+#: attention_tp / moe_tp / pdmux_prefill_tp carry no pynccl or custom
+#: all-reduce communicator, so a reduction dispatched to them asserts.
+_ELIGIBLE_GROUP_NAMES = frozenset({"tp"})
+
+
+@functools.lru_cache(maxsize=None)
+def _warn_deterministic_wins() -> None:
+ logger.warning(
+ "Deterministic inference is on; leaving PCIe-IPC all-reduce off. Its "
+ "kernels pick a tactic per shape, so the reduction order would follow "
+ "the batch shape."
+ )
+
+
+def eligible_group(
+ *, group_name: Optional[str], world_size: int, deterministic: bool
+) -> bool:
+ """Whether ``GroupCoordinator`` should build this backend for a group."""
+ if not (
+ envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.get()
+ and world_size > 1
+ and group_name in _ELIGIBLE_GROUP_NAMES
+ ):
+ return False
+ # Tactic is chosen per shape, so the reduction order would follow the
+ # batch shape; disabling tuning alone does not help, the seed policy is
+ # per-shape too.
+ if deterministic:
+ _warn_deterministic_wins()
+ return False
+ return True
+
+
+@functools.lru_cache(maxsize=None)
+def _warn_untuned_dtype(dtype: torch.dtype) -> None:
+ """Once per dtype: a per-layer warning would be issued thousands of times."""
+ logger.warning(
+ "FlashInfer PCIe-IPC all-reduce serves bf16 only; %s reductions stay on NCCL.",
+ dtype,
+ )
+
+
+def _autotune_disabled() -> bool:
+ """Whether the operator asked for no FlashInfer autotuning."""
+ from sglang.srt.runtime_context import get_exec
+
+ try:
+ return bool(get_exec().kernel.disable_flashinfer_autotune)
+ except ValueError:
+ return False
+
+
+def _decode_width() -> Optional[int]:
+ """Rows in the widest decode reduction, or None when the config is unavailable.
+
+ Decode runs inside a captured graph, so the largest captured batch bounds
+ the reduction. ``max_bs`` counts requests, not rows: a speculative forward
+ verifies several draft tokens per sequence, so the row count is
+ ``max_bs * draft tokens``, as in
+ ``runtime_context.cutedsl_moe_max_num_tokens``.
+ """
+ from sglang.srt.runtime_context import get_exec, get_spec
+
+ try:
+ cg_config = get_exec().graph.cuda_graph_config
+ spec = get_spec()
+ except ValueError:
+ # Embedded and unit-test use have no runtime context; the caller then
+ # sizes from a fixed width that ignores the server args, so say so.
+ logger.warning(
+ "FlashInfer PCIe-IPC has no runtime context to read the decode "
+ "width from; falling back to a fixed bound."
+ )
+ return None
+ if cg_config is None or not cg_config.decode.max_bs:
+ return None
+ rows_per_req = (
+ (spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1
+ )
+ return cg_config.decode.max_bs * rows_per_req
+
+
+def _tune_cache_path(world_size: int) -> Optional[str]:
+ """Where FlashInfer should persist the measured tactics.
+
+ Named explicitly rather than left to default, because the tuning run has to
+ write the file for the *next* server to reuse it; a workspace built without
+ a cache path measures every start.
+ """
+ try:
+ from flashinfer.comm.pcie_ipc_tuning import default_cache_path
+
+ return str(default_cache_path(world_size))
+ except (AttributeError, ImportError):
+ return None
+
+
+def _tune_batches(max_rows: int) -> tuple[int, ...]:
+ """Row counts to profile, bounded by what the workspace can actually take.
+
+ The autotuner measures one tactic per shape, so profiling rows the workspace
+ would reject is wasted startup time. Powers of two cover the captured decode
+ batches; ``max_rows`` is appended because it is the widest reduction served
+ and is not always a power of two.
+ """
+ rows = [r for r in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) if r <= max_rows]
+ if max_rows >= 1 and max_rows not in rows:
+ rows.append(max_rows)
+ return tuple(rows)
+
+
+class PcieIpcCommunicator:
+ """Adapter between ``GroupCoordinator`` and FlashInfer's PCIe-IPC all-reduce."""
+
+ def __init__(
+ self,
+ group: ProcessGroup,
+ device: torch.device | int,
+ cpu_group: Optional[ProcessGroup] = None,
+ ):
+ self.disabled = True
+ self.max_numel = 0
+ self._workspace: Optional[Any] = None
+ self._bound_stream: Optional[torch.cuda.Stream] = None
+ self._cpu_group = cpu_group
+ self._max_rows: Optional[int] = None
+
+ world_size = dist.get_world_size(group=group)
+ if world_size not in _SUPPORTED_WORLD_SIZES:
+ logger.debug(
+ "FlashInfer PCIe-IPC all-reduce skipped: unsupported world size %d",
+ world_size,
+ )
+ return
+
+ if isinstance(device, int):
+ device = torch.device(f"cuda:{device}")
+ if device.type != "cuda":
+ return
+
+ try:
+ from flashinfer.comm import PcieIpcAllReduceWorkspace
+ except ImportError:
+ logger.warning(
+ "SGLANG_ENABLE_PCIE_IPC_ALLREDUCE is set but this FlashInfer build has "
+ "no pcie_ipc_comm; falling back to NCCL all-reduce."
+ )
+ return
+
+ # The workspace is built on the first eligible tensor: sizing it needs the
+ # hidden size, which the group does not know yet.
+ self._group = group
+ self._device = device
+ self._world_size = world_size
+ self._workspace_cls = PcieIpcAllReduceWorkspace
+ self._build_failed = False
+ self.disabled = False
+ logger.info(
+ "FlashInfer PCIe-IPC all-reduce enabled (world=%d, workspace sized on "
+ "first reduction)",
+ world_size,
+ )
+
+ def _ensure_workspace(self, inp: torch.Tensor) -> bool:
+ """Build the workspace for the widest decode, using ``inp`` for the hidden size.
+
+ Every rank issues the same reductions in the same order, so they all arrive
+ here with the same shape and agree on ``max_numel`` without extra exchange.
+ """
+ if self._workspace is not None:
+ return True
+ if self._build_failed:
+ return False
+
+ hidden = inp.shape[-1]
+ override = envs.SGLANG_PCIE_IPC_MAX_NUMEL.get()
+ if override:
+ max_numel = override
+ else:
+ # The runner's width is already clamped to the request pool;
+ # _decode_width() can only re-derive it.
+ width = self._max_rows if self._max_rows else _decode_width()
+ if width is None:
+ # Admits a different set of batches than the server issues;
+ # every reduction outside it silently uses NCCL.
+ logger.warning(
+ "FlashInfer PCIe-IPC is sizing its workspace from the fixed "
+ "fallback width %d, not from this server's decode config; "
+ "set SGLANG_PCIE_IPC_MAX_NUMEL to override.",
+ _FALLBACK_DECODE_WIDTH,
+ )
+ width = _FALLBACK_DECODE_WIDTH
+ max_numel = width * hidden
+
+ try:
+ self._workspace = self._workspace_cls(
+ group=self._group,
+ max_numel=max_numel,
+ dtype=torch.bfloat16,
+ tune_batches=_tune_batches(max_numel // hidden),
+ tune_cache=_tune_cache_path(self._world_size),
+ )
+ except Exception as e:
+ logger.warning(
+ "FlashInfer PCIe-IPC workspace (max_numel=%d) failed: %s; "
+ "reductions stay on NCCL",
+ max_numel,
+ e,
+ )
+ self._build_failed = True
+ self.disabled = True
+ return False
+
+ self.max_numel = max_numel
+ logger.info(
+ "FlashInfer PCIe-IPC workspace built (world=%d, max_numel=%d, hidden=%d)",
+ self._world_size,
+ max_numel,
+ hidden,
+ )
+ self._tune(hidden)
+ return True
+
+ def prepare(self, *, hidden: int, max_rows: Optional[int] = None) -> None:
+ """Build and measure ahead of the first reduction.
+
+ Tuning runs here rather than on the first reduction, which may land
+ inside graph capture or another autotune pass; ``_tune`` declines in
+ both, leaving the kernels on FlashInfer's seed policy.
+
+ ``max_rows`` is the widest decode reduction the runner will issue.
+ """
+ if self.disabled or self._workspace is not None:
+ return
+ self._max_rows = max_rows
+ probe = torch.empty((1, hidden), dtype=torch.bfloat16, device=self._device)
+ self._ensure_workspace(probe)
+
+ def _tune(self, hidden: int) -> None:
+ """Measure the launch tactics for this workspace's shapes.
+
+ Without this the kernels run FlashInfer's seed policy, and ``supports``
+ answers from that policy rather than from measurements on this host.
+ Results persist to FlashInfer's cache, so only the first server pays.
+ """
+ if _autotune_disabled():
+ logger.info(
+ "FlashInfer PCIe-IPC all-reduce: --disable-flashinfer-autotune is "
+ "set; keeping FlashInfer's seed policy."
+ )
+ return
+
+ from flashinfer.autotuner import AutoTuner
+
+ if AutoTuner.get().is_tuning_mode:
+ # Reached through the lazy build, so tuning would start at
+ # whatever point the first reduction happens; keep it to prepare().
+ logger.warning(
+ "FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
+ "another autotune context; skipping autotune and keeping the seed "
+ "policy. Call PcieIpcCommunicator.prepare() from warmup to tune."
+ )
+ return
+ if self._cpu_group is None:
+ logger.warning(
+ "FlashInfer PCIe-IPC all-reduce has no host group to autotune on; "
+ "running FlashInfer's seed policy instead of measured tactics."
+ )
+ return
+ if torch.cuda.is_current_stream_capturing():
+ logger.warning(
+ "FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
+ "a graph capture; skipping autotune and keeping the seed policy."
+ )
+ return
+ try:
+ tuned = self._workspace.tune(
+ [hidden], dtype=torch.bfloat16, tune_group=self._cpu_group
+ )
+ except Exception as e:
+ logger.warning(
+ "FlashInfer PCIe-IPC autotune failed (%s); keeping the seed policy.",
+ e,
+ )
+ return
+ # Report what tune() covered, not that it returned: it declines shapes
+ # silently, and a log line that only proves no exception was raised is
+ # how an autotune that measured nothing reads as a success.
+ if not tuned:
+ logger.warning(
+ "FlashInfer PCIe-IPC autotune covered no shapes for hidden=%d "
+ "(requested rows %s); the kernels keep the seed policy.",
+ hidden,
+ _tune_batches(self.max_numel // hidden),
+ )
+ return
+ logger.info(
+ "FlashInfer PCIe-IPC all-reduce autotuned %d shape(s) (hidden=%d, rows=%s)",
+ len(tuned),
+ hidden,
+ _tune_batches(self.max_numel // hidden),
+ )
+
+ def should_pcie_ipc_ar(self, inp: torch.Tensor) -> bool:
+ """Whether this reduction should go to the kernels rather than NCCL.
+
+ ``supports`` answers whether the kernels can run the tensor, not
+ whether they beat NCCL on it, so the size bound above is what keeps
+ prefill-sized reductions on NCCL.
+ """
+ if self.disabled or not inp.is_contiguous() or inp.dim() < 2:
+ return False
+ # FlashInfer keys tuning results by dtype, and only bf16 is tuned here.
+ if inp.dtype is not torch.bfloat16:
+ _warn_untuned_dtype(inp.dtype)
+ return False
+ if not self._ensure_workspace(inp):
+ return False
+ if inp.numel() > self.max_numel:
+ return False
+ return self._workspace.supports(inp)
+
+ def pcie_ipc_all_reduce(self, inp: torch.Tensor) -> Optional[torch.Tensor]:
+ """All-reduce ``inp``, rebinding the workspace when the stream changes.
+
+ One workspace serves one stream: its epoch and arrival counters assume the
+ calls sharing it are totally ordered. SGLang switches streams at phase
+ boundaries -- graph capture, then replay -- so the binding is moved with
+ the caller rather than kept on whichever stream got there first. This is
+ safe because those phases do not overlap; it would not be safe if two
+ streams issued reductions concurrently on the same group.
+ """
+ stream = torch.cuda.current_stream()
+ if self._bound_stream is None or stream != self._bound_stream:
+ self._workspace.rebind_stream()
+ self._bound_stream = stream
+ return self._workspace.all_reduce(inp)
+
+ def destroy(self) -> None:
+ if self._workspace is not None:
+ self._workspace.destroy()
+ self._workspace = None
+ self.disabled = True
diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py
index 7676ef3676..37c6732744 100644
--- a/python/sglang/srt/distributed/parallel_state.py
+++ b/python/sglang/srt/distributed/parallel_state.py
@@ -485,6 +485,28 @@ class GroupCoordinator:
self.ca_comm: Optional[Any] = None
self.qr_comm: Optional[QuickAllReduce] = None
+
+ self.pcie_ipc_comm: Optional[Any] = None
+ from sglang.srt.distributed.device_communicators.pcie_ipc_ar import (
+ PcieIpcCommunicator,
+ eligible_group,
+ )
+
+ if eligible_group(
+ group_name=group_name,
+ world_size=self.world_size,
+ deterministic=self._deterministic_collectives_enabled(),
+ ):
+ try:
+ # The IPC handshake needs the CUDA (NCCL) group, not the CPU one.
+ # Autotuning is the other way round: it rendezvouses on the host.
+ self.pcie_ipc_comm = PcieIpcCommunicator(
+ group=self.device_group,
+ device=self.device,
+ cpu_group=self.cpu_group,
+ )
+ except Exception as e:
+ logger.warning(f"Setup FlashInfer PCIe-IPC allreduce failed with {e}.")
if use_custom_allreduce and self.world_size > 1:
# Initialize a custom fast all-reduce implementation.
try:
@@ -944,6 +966,16 @@ class GroupCoordinator:
and self.ca_comm.should_custom_ar(input_)
):
return "ca"
+ # After ``ca``: the PCIe-IPC kernels are for hosts where no fabric-specific
+ # backend applies. They do not probe for NVLink, so on a host that has it
+ # this ordering is what keeps the faster backend in front of them.
+ if (
+ self.pcie_ipc_comm is not None
+ and not self.pcie_ipc_comm.disabled
+ and not should_use_pymscclpp_allreduce
+ and self.pcie_ipc_comm.should_pcie_ipc_ar(input_)
+ ):
+ return "pcie_ipc"
if (
self.qr_comm is not None
and not self.qr_comm.disabled
@@ -1025,6 +1057,8 @@ class GroupCoordinator:
elif outplace_all_reduce_method == "pymscclpp":
assert not pymscclpp_comm.disabled
out = pymscclpp_comm.all_reduce(input_)
+ elif outplace_all_reduce_method == "pcie_ipc":
+ return self.pcie_ipc_comm.pcie_ipc_all_reduce(input_)
elif outplace_all_reduce_method == "pynccl":
with pynccl_comm.change_state(enable=True):
out = pynccl_comm.outplace_all_reduce(input_)
@@ -2059,6 +2093,11 @@ class GroupCoordinator:
return tensor
def destroy(self):
+ # Must precede destroy_process_group(): FlashInfer's workspace
+ # teardown collectives on the group it was built with.
+ if self.pcie_ipc_comm is not None:
+ self.pcie_ipc_comm.destroy()
+ self.pcie_ipc_comm = None
if self.device_group is not None:
torch.distributed.destroy_process_group(self.device_group)
self.device_group = None
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index 6566413a75..787234e08e 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -1350,6 +1350,9 @@ class Envs:
SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB = EnvInt(16 * 1024)
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PULL_SIZE_KB = EnvInt(None)
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PUSH_SIZE_KB = EnvInt(None)
+ # See docs/references/environment_variables.
+ SGLANG_ENABLE_PCIE_IPC_ALLREDUCE = EnvBool(False)
+ SGLANG_PCIE_IPC_MAX_NUMEL = EnvInt(0)
# ===================================================================
# RoPE cache
diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py
index a3a3f97245..643b7d0b63 100644
--- a/python/sglang/srt/model_executor/runner/base_runner.py
+++ b/python/sglang/srt/model_executor/runner/base_runner.py
@@ -253,6 +253,7 @@ class BaseRunner(ABC):
self._pre_initialize_flashinfer_allreduce_workspace()
self._pre_initialize_fi_a2a_workspace()
+ self._pre_initialize_pcie_ipc_workspace()
# Model-owned communication resources may depend on the resolved
# request pool and must be compiled/allocated before graph capture.
@@ -333,6 +334,44 @@ class BaseRunner(ABC):
init_fi_a2a_workspace(get_parallel().dcp_group)
+ def _pre_initialize_pcie_ipc_workspace(self):
+ """Build the PCIe-IPC all-reduce workspace before graph capture.
+
+ Runs for every model, not only the ones that autotune: left to the first
+ reduction, the build lands inside another autotune context and tuning
+ declines there.
+ """
+ pcie_ipc_comm = get_parallel().tp_group.pcie_ipc_comm
+ if pcie_ipc_comm is None:
+ return
+
+ mr = self.model_runner
+ pcie_ipc_comm.prepare(
+ hidden=mr.model_config.hidden_size,
+ max_rows=self._widest_decode_rows(),
+ )
+
+ def _widest_decode_rows(self) -> Optional[int]:
+ """Rows in the widest decode reduction, or None when no graph is captured.
+
+ Same derivation the decode runner uses for its buffers, so the workspace
+ covers exactly the batches that will be issued: get_batch_sizes_to_capture
+ already applies the attention-tp alignment and the req_to_token_pool clamp,
+ which a second derivation from cuda_graph_config would miss.
+ """
+ if get_exec().graph.cuda_graph_config is None:
+ return None
+ from sglang.srt.model_executor.runner.base_cuda_graph_runner import (
+ get_batch_sizes_to_capture,
+ )
+
+ mr = self.model_runner
+ tokens_per_req = mr.decode_num_tokens_per_req()
+ capture_bs, _ = get_batch_sizes_to_capture(mr, tokens_per_req)
+ if not capture_bs:
+ return None
+ return max(capture_bs) * tokens_per_req
+
def _flashinfer_autotune(self, *, buffers, batch_size):
"""Run flashinfer autotune.
diff --git a/test/registered/unit/distributed/test_pcie_ipc_ar.py b/test/registered/unit/distributed/test_pcie_ipc_ar.py
new file mode 100644
index 0000000000..3c06bf8450
--- /dev/null
+++ b/test/registered/unit/distributed/test_pcie_ipc_ar.py
@@ -0,0 +1,513 @@
+"""Unit tests for the FlashInfer PCIe-IPC all-reduce adapter.
+
+These cover the parts that decide whether a reduction reaches the kernels at
+all -- world-size gating, workspace sizing, and the shape guard -- because each
+of them fails *silently* in production: an unsupported shape is not an error,
+it is a fall back to NCCL, which looks exactly like the kernels being slow.
+"""
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+import torch
+
+from sglang.srt.distributed.device_communicators import pcie_ipc_ar
+from sglang.srt.distributed.device_communicators.pcie_ipc_ar import (
+ PcieIpcCommunicator,
+)
+from sglang.srt.environ import envs
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+
+HIDDEN = 6144
+
+
+def _make_comm(
+ world_size=8, workspace_cls=None, cpu_group=None, tune=None, max_rows=None
+):
+ """Build a communicator with the collaborators stubbed out.
+
+ The constructor is bypassed: it needs a live process group and a CUDA
+ device, neither of which this test has, and neither of which the logic
+ under test depends on.
+
+ ``_ensure_workspace`` ends by calling ``_tune``, which imports FlashInfer
+ before it reads ``_cpu_group``. Neither belongs to the sizing or shape
+ logic these helpers serve, and the import fails outright on a CPU runner,
+ so tuning is stubbed by default and covered on its own in ``TestTuning``.
+ """
+ comm = PcieIpcCommunicator.__new__(PcieIpcCommunicator)
+ comm.disabled = False
+ comm.max_numel = 0
+ comm._workspace = None
+ comm._bound_stream = None
+ comm._group = MagicMock()
+ comm._device = torch.device("cpu")
+ comm._world_size = world_size
+ comm._workspace_cls = workspace_cls or MagicMock()
+ comm._build_failed = False
+ comm._cpu_group = cpu_group
+ comm._max_rows = max_rows
+ comm._tune = tune if tune is not None else (lambda hidden: None)
+ return comm
+
+
+class TestWorldSizeGate(CustomTestCase):
+ def test_unsupported_world_size_disables(self):
+ """A world size the kernels have no IPC channels for must disable cleanly."""
+ for world_size in (1, 3, 5, 6, 7, 16):
+ with self.subTest(world_size=world_size):
+ with patch.object(
+ pcie_ipc_ar.dist, "get_world_size", return_value=world_size
+ ):
+ comm = PcieIpcCommunicator(group=MagicMock(), device=0)
+ self.assertTrue(comm.disabled)
+
+ def test_missing_flashinfer_disables_instead_of_raising(self):
+ """A build without pcie_ipc_comm must degrade to NCCL, not crash the server."""
+ with (
+ patch.object(pcie_ipc_ar.dist, "get_world_size", return_value=8),
+ patch("builtins.__import__", side_effect=ImportError("no pcie_ipc_comm")),
+ ):
+ comm = PcieIpcCommunicator(group=MagicMock(), device=0)
+ self.assertTrue(comm.disabled)
+
+
+class TestGroupEligibility(CustomTestCase):
+ """Which groups get this backend, which is the rule that broke other models.
+
+ ``"tp" in unique_name`` also matched attention_tp / moe_tp /
+ pdmux_prefill_tp. Those groups carry no pynccl or custom all-reduce
+ communicator, so a reduction dispatched to them hits an assertion instead
+ of falling back, and Qwen3-8B TP4/DP2 and Qwen3-30B-A3B TP4/EP2 fail to
+ start with the flag on.
+ """
+
+ def test_only_the_tensor_parallel_group_is_eligible(self):
+ with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True):
+ self.assertTrue(
+ pcie_ipc_ar.eligible_group(
+ group_name="tp", world_size=4, deterministic=False
+ )
+ )
+ for name in ("attention_tp", "moe_tp", "pdmux_prefill_tp", "world", "pp"):
+ with self.subTest(group=name):
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name=name, world_size=4, deterministic=False
+ )
+ )
+
+ def test_deterministic_inference_keeps_the_backend_off(self):
+ """Deterministic inference forbids a shape-dependent reduction order.
+
+ The kernels select a tactic per shape, so the order would follow the
+ batch shape -- which is why that mode already pins NCCL and turns off
+ custom and symmetric-memory all-reduce. Disabling tuning is not enough:
+ FlashInfer's seed policy is per-shape as well.
+ """
+ with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True):
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ pcie_ipc_ar._warn_deterministic_wins.cache_clear()
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name="tp", world_size=4, deterministic=True
+ )
+ )
+ self.assertIn("Deterministic inference", "\n".join(logs.output))
+
+ def test_the_deterministic_predicate_still_exists(self):
+ """The caller supplies this flag, so a mock cannot catch a rename.
+
+ GroupCoordinator owns the predicate -- it also folds in the
+ SGLANG_USE_1STAGE_ALLREDUCE override -- and this backend must keep
+ using that one rather than reading the env var a second time.
+ """
+ from sglang.srt.distributed.parallel_state import GroupCoordinator
+
+ self.assertTrue(hasattr(GroupCoordinator, "_deterministic_collectives_enabled"))
+
+ def test_single_rank_group_is_not_eligible(self):
+ with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True):
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name="tp", world_size=1, deterministic=False
+ )
+ )
+
+ def test_disabled_by_default(self):
+ """The backend is opt-in; nothing attaches without the flag."""
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name="tp", world_size=4, deterministic=False
+ )
+ )
+
+ def test_anonymous_group_is_not_eligible(self):
+ with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True):
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name=None, world_size=4, deterministic=False
+ )
+ )
+ self.assertFalse(
+ pcie_ipc_ar.eligible_group(
+ group_name="anonymous", world_size=4, deterministic=False
+ )
+ )
+
+
+class TestWorkspaceRelease(CustomTestCase):
+ """The workspace must be released before the groups it was built on.
+
+ FlashInfer collectives on that group while tearing the workspace down, so
+ ordering is the whole point: releasing afterwards is not a leak that shows
+ up in testing, it is a collective on a destroyed group.
+ """
+
+ def _coordinator(self, order):
+ from sglang.srt.distributed.parallel_state import GroupCoordinator
+
+ coordinator = GroupCoordinator.__new__(GroupCoordinator)
+ coordinator.device_group = MagicMock(name="device_group")
+ coordinator.cpu_group = MagicMock(name="cpu_group")
+ coordinator.pynccl_comm = None
+ coordinator.pymscclpp_comm = None
+ coordinator.ca_comm = None
+ coordinator.mq_broadcaster = None
+ workspace = MagicMock()
+ workspace.destroy.side_effect = lambda: order.append("workspace")
+ coordinator.pcie_ipc_comm = workspace
+ return coordinator
+
+ def test_released_before_the_process_groups(self):
+ order = []
+ coordinator = self._coordinator(order)
+ with patch.object(
+ torch.distributed,
+ "destroy_process_group",
+ side_effect=lambda g: order.append("process_group"),
+ ):
+ coordinator.destroy()
+
+ self.assertEqual(order[0], "workspace")
+ self.assertEqual(order.count("process_group"), 2)
+ self.assertIsNone(coordinator.pcie_ipc_comm)
+
+ def test_destroy_without_the_backend_is_a_no_op(self):
+ order = []
+ coordinator = self._coordinator(order)
+ coordinator.pcie_ipc_comm = None
+ with patch.object(
+ torch.distributed,
+ "destroy_process_group",
+ side_effect=lambda g: order.append("process_group"),
+ ):
+ coordinator.destroy()
+ self.assertEqual(order, ["process_group", "process_group"])
+
+
+class TestWorkspaceSizing(CustomTestCase):
+ def test_sized_for_decode_not_prefill(self):
+ """The default bound is the decode width, so prefill chunks stay on NCCL.
+
+ Sizing for a prefill chunk was measured 66% worse on TTFT and bought
+ nothing on TPOT, so this is the behaviour that must not regress.
+ """
+ comm = _make_comm()
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=64):
+ self.assertTrue(
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ )
+ self.assertEqual(comm.max_numel, 64 * HIDDEN)
+
+ prefill = torch.empty(16384, HIDDEN, dtype=torch.bfloat16)
+ self.assertGreater(prefill.numel(), comm.max_numel)
+ self.assertFalse(comm.should_pcie_ipc_ar(prefill))
+
+ def test_falls_back_when_server_args_absent(self):
+ """Embedded/unit use has no server args; the bound must still be finite."""
+ comm = _make_comm()
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=None):
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ self.assertEqual(comm.max_numel, pcie_ipc_ar._FALLBACK_DECODE_WIDTH * HIDDEN)
+
+ def test_env_override_wins(self):
+ comm = _make_comm()
+ with envs.SGLANG_PCIE_IPC_MAX_NUMEL.override(123456):
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=64):
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ self.assertEqual(comm.max_numel, 123456)
+
+ def test_build_failure_disables_once(self):
+ """A workspace that cannot be allocated must not be retried every layer."""
+ cls = MagicMock(side_effect=RuntimeError("out of IPC handles"))
+ comm = _make_comm(workspace_cls=cls)
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=64):
+ self.assertFalse(
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ )
+ self.assertFalse(
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ )
+ self.assertTrue(comm.disabled)
+ self.assertEqual(cls.call_count, 1)
+
+
+class TestShapeGuard(CustomTestCase):
+ def _ready_comm(self):
+ comm = _make_comm()
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=64):
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ comm._workspace.supports.return_value = True
+ return comm
+
+ def test_delegates_to_flashinfer_supports(self):
+ """Coverage is FlashInfer's decision; a rejected shape keeps the NCCL path."""
+ comm = self._ready_comm()
+ inp = torch.empty(4, HIDDEN, dtype=torch.bfloat16)
+ self.assertTrue(comm.should_pcie_ipc_ar(inp))
+
+ comm._workspace.supports.return_value = False
+ self.assertFalse(comm.should_pcie_ipc_ar(inp))
+
+ def test_rejects_noncontiguous_and_1d(self):
+ comm = self._ready_comm()
+ self.assertFalse(
+ comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=torch.bfloat16).t())
+ )
+ self.assertFalse(
+ comm.should_pcie_ipc_ar(torch.empty(HIDDEN, dtype=torch.bfloat16))
+ )
+
+ def test_rejects_dtypes_that_were_never_tuned(self):
+ """FlashInfer keys tuning results by dtype, so fp16 would run untuned.
+
+ This adapter only ever tunes bf16, so anything else must keep its NCCL
+ path rather than run the seed policy while looking tuned.
+ """
+ comm = self._ready_comm()
+ for dtype in (torch.float16, torch.float32):
+ with self.subTest(dtype=dtype):
+ self.assertFalse(
+ comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=dtype))
+ )
+
+ def test_disabled_communicator_never_claims_a_tensor(self):
+ comm = self._ready_comm()
+ comm.disabled = True
+ self.assertFalse(
+ comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=torch.bfloat16))
+ )
+
+
+class TestDecodeWidth(CustomTestCase):
+ """The width must come from the server's own decode config.
+
+ Reading it from ``ServerArgs`` raised ``AttributeError`` on every call once
+ upstream moved the resolved config onto the exec bag, and a bare ``except``
+ turned that into a permanent, silent fallback: whatever
+ ``--cuda-graph-max-bs-decode``
+ the operator passed, the workspace was sized for a fixed 64 rows.
+ """
+
+ def _context(self, max_bs, algorithm=None, draft_tokens=None):
+ cg = MagicMock()
+ cg.decode.max_bs = max_bs
+ exec_bag = MagicMock()
+ exec_bag.graph.cuda_graph_config = cg
+ spec = MagicMock()
+ spec.speculative_algorithm = algorithm
+ spec.speculative_num_draft_tokens = draft_tokens
+ return {
+ "sglang.srt.runtime_context": MagicMock(
+ get_exec=lambda: exec_bag, get_spec=lambda: spec
+ )
+ }
+
+ def test_width_follows_the_resolved_decode_max_bs(self):
+ with patch.dict("sys.modules", self._context(max_bs=128)):
+ self.assertEqual(pcie_ipc_ar._decode_width(), 128)
+
+ def test_width_multiplies_by_the_speculative_draft_width(self):
+ """max_bs counts requests; a verify pass carries several rows each."""
+ with patch.dict(
+ "sys.modules", self._context(max_bs=64, algorithm="EAGLE", draft_tokens=4)
+ ):
+ self.assertEqual(pcie_ipc_ar._decode_width(), 256)
+
+ def test_no_speculation_means_one_row_per_request(self):
+ with patch.dict(
+ "sys.modules", self._context(max_bs=64, algorithm=None, draft_tokens=4)
+ ):
+ self.assertEqual(pcie_ipc_ar._decode_width(), 64)
+
+ def test_absent_config_is_reported_not_swallowed(self):
+ """The fallback is fine; being quiet about it is what hid this."""
+ broken = MagicMock()
+ broken.graph.cuda_graph_config = None
+ with patch.dict(
+ "sys.modules",
+ {"sglang.srt.runtime_context": MagicMock(get_exec=lambda: broken)},
+ ):
+ self.assertIsNone(pcie_ipc_ar._decode_width())
+
+ def test_the_attribute_path_it_reads_still_exists(self):
+ """Mocks cannot catch a field that upstream renamed or moved.
+
+ The defect this class covers was exactly that: the derivation kept
+ reading a path that had stopped existing, and the tests around it were
+ all mocked, so nothing noticed. Pin the real chain.
+ """
+ from sglang.srt import runtime_context
+ from sglang.srt.model_executor.cuda_graph_config import (
+ default_cuda_graph_config,
+ )
+
+ self.assertTrue(hasattr(runtime_context, "get_exec"))
+ self.assertTrue(hasattr(runtime_context, "get_spec"))
+ self.assertTrue(hasattr(default_cuda_graph_config().decode, "max_bs"))
+
+ def test_runner_width_wins_over_the_re_derivation(self):
+ """prepare() carries the width the decode graphs were captured for.
+
+ get_batch_sizes_to_capture already applies the attention-tp alignment
+ and the req_to_token_pool clamp; _decode_width() cannot see either, so
+ when the runner supplies a width it has to take precedence.
+ """
+ comm = _make_comm(max_rows=24)
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=512):
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ self.assertEqual(comm.max_numel, 24 * HIDDEN)
+
+ def test_fallback_width_is_announced(self):
+ comm = _make_comm()
+ with patch.object(pcie_ipc_ar, "_decode_width", return_value=None):
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16))
+ self.assertIn("fallback width", "\n".join(logs.output))
+ self.assertEqual(comm.max_numel, pcie_ipc_ar._FALLBACK_DECODE_WIDTH * HIDDEN)
+
+
+class TestTuning(CustomTestCase):
+ """Every way tuning declines leaves the kernels on FlashInfer's seed policy.
+
+ ``tune()`` returning is not evidence it measured anything: it declines
+ shapes silently, and the three guards below return before calling it at
+ all. That is indistinguishable from a successful tune unless asserted,
+ which is how two earlier revisions of this adapter were misread.
+ """
+
+ def _comm(self, cpu_group=object(), tuned=("shape",)):
+ comm = _make_comm(cpu_group=cpu_group)
+ comm._workspace = MagicMock()
+ comm._workspace.tune.return_value = list(tuned)
+ comm.max_numel = 64 * HIDDEN
+ return comm
+
+ @staticmethod
+ def _flashinfer(is_tuning_mode=False):
+ """Stand in for FlashInfer, which a CPU runner does not have installed."""
+ autotuner = MagicMock()
+ autotuner.AutoTuner.get.return_value.is_tuning_mode = is_tuning_mode
+ return {"flashinfer": MagicMock(), "flashinfer.autotuner": autotuner}
+
+ def _run_tune(self, comm, is_tuning_mode=False, capturing=False):
+ with (
+ patch.dict("sys.modules", self._flashinfer(is_tuning_mode)),
+ patch.object(
+ torch.cuda, "is_current_stream_capturing", return_value=capturing
+ ),
+ ):
+ PcieIpcCommunicator._tune(comm, HIDDEN)
+
+ def test_autotune_disabled_reads_the_real_flag(self):
+ """Exercise the helper itself, not a patched stand-in.
+
+ Patching _autotune_disabled in the tests above cannot catch a wrong or
+ renamed field inside it -- the same blind spot that let _decode_width
+ read a path that had stopped existing.
+ """
+ for flag in (True, False):
+ with self.subTest(flag=flag):
+ bag = MagicMock()
+ bag.kernel.disable_flashinfer_autotune = flag
+ with patch.dict(
+ "sys.modules",
+ {"sglang.srt.runtime_context": MagicMock(get_exec=lambda: bag)},
+ ):
+ self.assertIs(pcie_ipc_ar._autotune_disabled(), flag)
+
+ def test_the_flag_still_exists_on_the_exec_bag(self):
+ from sglang.srt.arg_groups.fields.exec_ import ExecKernel
+
+ self.assertIn("disable_flashinfer_autotune", ExecKernel.__annotations__)
+
+ def test_honours_disable_flashinfer_autotune(self):
+ """--disable-flashinfer-autotune must stop this tuning path too.
+
+ The flag gates should_run_flashinfer_autotune, which gates prepare().
+ The lazy build on the first reduction is a second way in, and it used to
+ tune and write a cache while the operator had asked for neither.
+ """
+ comm = self._comm()
+ with patch.object(pcie_ipc_ar, "_autotune_disabled", return_value=True):
+ with self.assertLogs(pcie_ipc_ar.logger, level="INFO") as logs:
+ self._run_tune(comm)
+ self.assertIn("--disable-flashinfer-autotune", "\n".join(logs.output))
+ comm._workspace.tune.assert_not_called()
+
+ def test_declines_inside_another_autotune_context(self):
+ """FlashInfer will not profile a collective from a context it did not open."""
+ comm = self._comm()
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ self._run_tune(comm, is_tuning_mode=True)
+ self.assertIn("another autotune context", "\n".join(logs.output))
+ comm._workspace.tune.assert_not_called()
+
+ def test_declines_without_a_host_group(self):
+ """The autotuner rendezvouses on the host; no CPU group means no measurement."""
+ comm = self._comm(cpu_group=None)
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ self._run_tune(comm)
+ self.assertIn("no host group", "\n".join(logs.output))
+ comm._workspace.tune.assert_not_called()
+
+ def test_declines_under_graph_capture(self):
+ """Autotuning replays kernels, which cannot happen inside a capture."""
+ comm = self._comm()
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ self._run_tune(comm, capturing=True)
+ self.assertIn("capture", "\n".join(logs.output).lower())
+ comm._workspace.tune.assert_not_called()
+
+ def test_warns_when_tune_covered_no_shapes(self):
+ """tune() declining every shape must not read as a successful tune."""
+ comm = self._comm(tuned=())
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs:
+ self._run_tune(comm)
+ comm._workspace.tune.assert_called_once()
+ self.assertIn("covered no shapes", "\n".join(logs.output))
+
+ def test_reports_the_shapes_it_measured(self):
+ comm = self._comm(tuned=("a", "b", "c"))
+ with self.assertLogs(pcie_ipc_ar.logger, level="INFO") as logs:
+ self._run_tune(comm)
+ comm._workspace.tune.assert_called_once()
+ _, kwargs = comm._workspace.tune.call_args
+ self.assertEqual(kwargs["dtype"], torch.bfloat16)
+ self.assertIs(kwargs["tune_group"], comm._cpu_group)
+ self.assertIn("autotuned 3 shape(s)", "\n".join(logs.output))
+
+ def test_tune_failure_keeps_the_seed_policy(self):
+ """A raising autotuner must not take the server down with it."""
+ comm = self._comm()
+ comm._workspace.tune.side_effect = RuntimeError("nvlink probe failed")
+ with self.assertLogs(pcie_ipc_ar.logger, level="WARNING"):
+ self._run_tune(comm)
+
+
+if __name__ == "__main__":
+ unittest.main()