From c7be3e935b5034006cd6ae7977b41e2459b4126c Mon Sep 17 00:00:00 2001 From: eeecho <82921722+AliceChenyy@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:08:44 +0800 Subject: [PATCH] [SM120] Add optional FlashInfer PCIe-IPC all-reduce for switch-free hosts (#34528) --- .../docs/references/environment_variables.mdx | 10 + .../device_communicators/pcie_ipc_ar.py | 426 +++++++++++++++ .../sglang/srt/distributed/parallel_state.py | 39 ++ python/sglang/srt/environ.py | 3 + .../srt/model_executor/runner/base_runner.py | 39 ++ .../unit/distributed/test_pcie_ipc_ar.py | 513 ++++++++++++++++++ 6 files changed, 1030 insertions(+) create mode 100644 python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py create mode 100644 test/registered/unit/distributed/test_pcie_ipc_ar.py diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index b930c847f0..24ca0f0861 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -1705,6 +1705,16 @@ SGLang supports various environment variables that can be used to configure its Use the 1-stage all-reduce kernel on AMD (deterministic, fixed accumulation order). If unset, it is auto-enabled when deterministic inference is on. false + + SGLANG_ENABLE_PCIE_IPC_ALLREDUCE + Use FlashInfer's PCIe-IPC all-reduce on switch-free intra-node hosts (no NVLink, no multicast), where the other custom all-reduce backends do not apply. Serves bf16 decode reductions that fit the workspace bound; everything else stays on NCCL. Left off under deterministic inference. + false + + + SGLANG_PCIE_IPC_MAX_NUMEL + Elements the PCIe-IPC workspace is sized for; it cannot grow after construction. 0 derives it from the widest decode batch the runner captures for. Raising it hands larger reductions to the kernels, at roughly 2 * world_size * max_numel * itemsize bytes per rank. + 0 + SGLANG_FLASHINFER_PREFILL_SPLIT_TILE_SIZE FlashInfer prefill split-tile size for deterministic attention. diff --git a/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py b/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py new file mode 100644 index 0000000000..6a8b36db69 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/pcie_ipc_ar.py @@ -0,0 +1,426 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FlashInfer PCIe-IPC all-reduce for switch-free intra-node machines. + +FlashInfer's ``pcie_ipc_comm`` targets hosts where every peer transfer crosses the +CPU root complex -- no NVLink, no multicast. The existing custom all-reduce +backends assume one of those fabrics, so on such a host every per-layer reduction +falls back to NCCL, which leaves a large margin on the table at decode sizes. + +Why the kernel wins here +------------------------ +Naive all-to-all peer writes collapse on this fabric, and the collapse worsens +with the number of concurrently writing blocks. FlashInfer's kernels instead +*stage* their pushes so each rank has exactly one outbound and one inbound stream +at any instant, and the 8-rank path keeps a 4+4 island decomposition so the +scarce cross-socket link carries the minimum. Measured on one 8x SM120 host +(GPUs 0-3 and 4-7 on separate NUMA nodes), against NCCL, hidden 6144 bf16: + +========= =========== ========== ========= +Message FlashInfer NCCL Speed-up +========= =========== ========== ========= +12 KiB 5.3 us 205.1 us 38.8x +96 KiB 15.3 us 584.9 us 38.2x +384 KiB 34.6 us 1169.3 us 33.8x +100 MiB 8.5 ms 50.4 ms 5.9x +========= =========== ========== ========= + +:meth:`PcieIpcAllReduceWorkspace.supports` reports whether the kernels can run +a shape, not whether they beat NCCL on it, so the workspace bound below is what +keeps the reductions this backend should not serve on NCCL. + +Workspace sizing +---------------- +The workspace is sized for the widest decode reduction the runner will issue, +``max_bs * draft_tokens * hidden``. The bound is a row count, not a phase +split: a prefill chunk narrower than it is admitted too. What the bound rules +out is the full-width prefill chunk, which is orders of magnitude larger, and +there NCCL's ring is the better algorithm. Measured at 8 ranks, TP8, 8k +context, against NCCL as the baseline: + +=================== ========== ========== =============== +Workspace TTFT TPOT Output tok/s +=================== ========== ========== =============== +NCCL only 1910 ms 21.14 ms 35.02 +prefill-sized 3176 ms 13.64 ms 38.43 +decode-sized 1849 ms 13.62 ms 48.05 +=================== ========== ========== =============== + +Handing prefill to these kernels costs 66% on TTFT and buys nothing on TPOT. +The decode-sized workspace keeps the whole TPOT win, stays level with NCCL on +TTFT, and is the only one of the three that is not worse than NCCL anywhere we +measured. It is also ~250x smaller, which matters at long context: at 128k with +4 concurrent requests the prefill-sized workspace regressed TPOT by 45%, and +that regression disappears when it is sized for decode. + +The bound comes from the resolved decode config, but the hidden size is not +known when the group is built, so the workspace is created on the first +eligible tensor, whose trailing dimension is exactly that hidden size. Ranks +run the same sequence of reductions, so they all reach that first call with the +same shape and build the same workspace. ``SGLANG_PCIE_IPC_MAX_NUMEL`` overrides +the derivation for deployments that want to hand larger reductions to the +kernels anyway. +""" + +import functools +import logging +from typing import Any, Optional + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.environ import envs + +logger = logging.getLogger(__name__) + +# The kernels build IPC channels for these world sizes only. +_SUPPORTED_WORLD_SIZES = (2, 4, 8) + +# Fallback decode width when the server args are absent (unit tests, embedded +# use). Covers the batch sizes we serve; anything larger stays on NCCL. +_FALLBACK_DECODE_WIDTH = 64 + + +#: attention_tp / moe_tp / pdmux_prefill_tp carry no pynccl or custom +#: all-reduce communicator, so a reduction dispatched to them asserts. +_ELIGIBLE_GROUP_NAMES = frozenset({"tp"}) + + +@functools.lru_cache(maxsize=None) +def _warn_deterministic_wins() -> None: + logger.warning( + "Deterministic inference is on; leaving PCIe-IPC all-reduce off. Its " + "kernels pick a tactic per shape, so the reduction order would follow " + "the batch shape." + ) + + +def eligible_group( + *, group_name: Optional[str], world_size: int, deterministic: bool +) -> bool: + """Whether ``GroupCoordinator`` should build this backend for a group.""" + if not ( + envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.get() + and world_size > 1 + and group_name in _ELIGIBLE_GROUP_NAMES + ): + return False + # Tactic is chosen per shape, so the reduction order would follow the + # batch shape; disabling tuning alone does not help, the seed policy is + # per-shape too. + if deterministic: + _warn_deterministic_wins() + return False + return True + + +@functools.lru_cache(maxsize=None) +def _warn_untuned_dtype(dtype: torch.dtype) -> None: + """Once per dtype: a per-layer warning would be issued thousands of times.""" + logger.warning( + "FlashInfer PCIe-IPC all-reduce serves bf16 only; %s reductions stay on NCCL.", + dtype, + ) + + +def _autotune_disabled() -> bool: + """Whether the operator asked for no FlashInfer autotuning.""" + from sglang.srt.runtime_context import get_exec + + try: + return bool(get_exec().kernel.disable_flashinfer_autotune) + except ValueError: + return False + + +def _decode_width() -> Optional[int]: + """Rows in the widest decode reduction, or None when the config is unavailable. + + Decode runs inside a captured graph, so the largest captured batch bounds + the reduction. ``max_bs`` counts requests, not rows: a speculative forward + verifies several draft tokens per sequence, so the row count is + ``max_bs * draft tokens``, as in + ``runtime_context.cutedsl_moe_max_num_tokens``. + """ + from sglang.srt.runtime_context import get_exec, get_spec + + try: + cg_config = get_exec().graph.cuda_graph_config + spec = get_spec() + except ValueError: + # Embedded and unit-test use have no runtime context; the caller then + # sizes from a fixed width that ignores the server args, so say so. + logger.warning( + "FlashInfer PCIe-IPC has no runtime context to read the decode " + "width from; falling back to a fixed bound." + ) + return None + if cg_config is None or not cg_config.decode.max_bs: + return None + rows_per_req = ( + (spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1 + ) + return cg_config.decode.max_bs * rows_per_req + + +def _tune_cache_path(world_size: int) -> Optional[str]: + """Where FlashInfer should persist the measured tactics. + + Named explicitly rather than left to default, because the tuning run has to + write the file for the *next* server to reuse it; a workspace built without + a cache path measures every start. + """ + try: + from flashinfer.comm.pcie_ipc_tuning import default_cache_path + + return str(default_cache_path(world_size)) + except (AttributeError, ImportError): + return None + + +def _tune_batches(max_rows: int) -> tuple[int, ...]: + """Row counts to profile, bounded by what the workspace can actually take. + + The autotuner measures one tactic per shape, so profiling rows the workspace + would reject is wasted startup time. Powers of two cover the captured decode + batches; ``max_rows`` is appended because it is the widest reduction served + and is not always a power of two. + """ + rows = [r for r in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) if r <= max_rows] + if max_rows >= 1 and max_rows not in rows: + rows.append(max_rows) + return tuple(rows) + + +class PcieIpcCommunicator: + """Adapter between ``GroupCoordinator`` and FlashInfer's PCIe-IPC all-reduce.""" + + def __init__( + self, + group: ProcessGroup, + device: torch.device | int, + cpu_group: Optional[ProcessGroup] = None, + ): + self.disabled = True + self.max_numel = 0 + self._workspace: Optional[Any] = None + self._bound_stream: Optional[torch.cuda.Stream] = None + self._cpu_group = cpu_group + self._max_rows: Optional[int] = None + + world_size = dist.get_world_size(group=group) + if world_size not in _SUPPORTED_WORLD_SIZES: + logger.debug( + "FlashInfer PCIe-IPC all-reduce skipped: unsupported world size %d", + world_size, + ) + return + + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + if device.type != "cuda": + return + + try: + from flashinfer.comm import PcieIpcAllReduceWorkspace + except ImportError: + logger.warning( + "SGLANG_ENABLE_PCIE_IPC_ALLREDUCE is set but this FlashInfer build has " + "no pcie_ipc_comm; falling back to NCCL all-reduce." + ) + return + + # The workspace is built on the first eligible tensor: sizing it needs the + # hidden size, which the group does not know yet. + self._group = group + self._device = device + self._world_size = world_size + self._workspace_cls = PcieIpcAllReduceWorkspace + self._build_failed = False + self.disabled = False + logger.info( + "FlashInfer PCIe-IPC all-reduce enabled (world=%d, workspace sized on " + "first reduction)", + world_size, + ) + + def _ensure_workspace(self, inp: torch.Tensor) -> bool: + """Build the workspace for the widest decode, using ``inp`` for the hidden size. + + Every rank issues the same reductions in the same order, so they all arrive + here with the same shape and agree on ``max_numel`` without extra exchange. + """ + if self._workspace is not None: + return True + if self._build_failed: + return False + + hidden = inp.shape[-1] + override = envs.SGLANG_PCIE_IPC_MAX_NUMEL.get() + if override: + max_numel = override + else: + # The runner's width is already clamped to the request pool; + # _decode_width() can only re-derive it. + width = self._max_rows if self._max_rows else _decode_width() + if width is None: + # Admits a different set of batches than the server issues; + # every reduction outside it silently uses NCCL. + logger.warning( + "FlashInfer PCIe-IPC is sizing its workspace from the fixed " + "fallback width %d, not from this server's decode config; " + "set SGLANG_PCIE_IPC_MAX_NUMEL to override.", + _FALLBACK_DECODE_WIDTH, + ) + width = _FALLBACK_DECODE_WIDTH + max_numel = width * hidden + + try: + self._workspace = self._workspace_cls( + group=self._group, + max_numel=max_numel, + dtype=torch.bfloat16, + tune_batches=_tune_batches(max_numel // hidden), + tune_cache=_tune_cache_path(self._world_size), + ) + except Exception as e: + logger.warning( + "FlashInfer PCIe-IPC workspace (max_numel=%d) failed: %s; " + "reductions stay on NCCL", + max_numel, + e, + ) + self._build_failed = True + self.disabled = True + return False + + self.max_numel = max_numel + logger.info( + "FlashInfer PCIe-IPC workspace built (world=%d, max_numel=%d, hidden=%d)", + self._world_size, + max_numel, + hidden, + ) + self._tune(hidden) + return True + + def prepare(self, *, hidden: int, max_rows: Optional[int] = None) -> None: + """Build and measure ahead of the first reduction. + + Tuning runs here rather than on the first reduction, which may land + inside graph capture or another autotune pass; ``_tune`` declines in + both, leaving the kernels on FlashInfer's seed policy. + + ``max_rows`` is the widest decode reduction the runner will issue. + """ + if self.disabled or self._workspace is not None: + return + self._max_rows = max_rows + probe = torch.empty((1, hidden), dtype=torch.bfloat16, device=self._device) + self._ensure_workspace(probe) + + def _tune(self, hidden: int) -> None: + """Measure the launch tactics for this workspace's shapes. + + Without this the kernels run FlashInfer's seed policy, and ``supports`` + answers from that policy rather than from measurements on this host. + Results persist to FlashInfer's cache, so only the first server pays. + """ + if _autotune_disabled(): + logger.info( + "FlashInfer PCIe-IPC all-reduce: --disable-flashinfer-autotune is " + "set; keeping FlashInfer's seed policy." + ) + return + + from flashinfer.autotuner import AutoTuner + + if AutoTuner.get().is_tuning_mode: + # Reached through the lazy build, so tuning would start at + # whatever point the first reduction happens; keep it to prepare(). + logger.warning( + "FlashInfer PCIe-IPC all-reduce reached its first reduction inside " + "another autotune context; skipping autotune and keeping the seed " + "policy. Call PcieIpcCommunicator.prepare() from warmup to tune." + ) + return + if self._cpu_group is None: + logger.warning( + "FlashInfer PCIe-IPC all-reduce has no host group to autotune on; " + "running FlashInfer's seed policy instead of measured tactics." + ) + return + if torch.cuda.is_current_stream_capturing(): + logger.warning( + "FlashInfer PCIe-IPC all-reduce reached its first reduction inside " + "a graph capture; skipping autotune and keeping the seed policy." + ) + return + try: + tuned = self._workspace.tune( + [hidden], dtype=torch.bfloat16, tune_group=self._cpu_group + ) + except Exception as e: + logger.warning( + "FlashInfer PCIe-IPC autotune failed (%s); keeping the seed policy.", + e, + ) + return + # Report what tune() covered, not that it returned: it declines shapes + # silently, and a log line that only proves no exception was raised is + # how an autotune that measured nothing reads as a success. + if not tuned: + logger.warning( + "FlashInfer PCIe-IPC autotune covered no shapes for hidden=%d " + "(requested rows %s); the kernels keep the seed policy.", + hidden, + _tune_batches(self.max_numel // hidden), + ) + return + logger.info( + "FlashInfer PCIe-IPC all-reduce autotuned %d shape(s) (hidden=%d, rows=%s)", + len(tuned), + hidden, + _tune_batches(self.max_numel // hidden), + ) + + def should_pcie_ipc_ar(self, inp: torch.Tensor) -> bool: + """Whether this reduction should go to the kernels rather than NCCL. + + ``supports`` answers whether the kernels can run the tensor, not + whether they beat NCCL on it, so the size bound above is what keeps + prefill-sized reductions on NCCL. + """ + if self.disabled or not inp.is_contiguous() or inp.dim() < 2: + return False + # FlashInfer keys tuning results by dtype, and only bf16 is tuned here. + if inp.dtype is not torch.bfloat16: + _warn_untuned_dtype(inp.dtype) + return False + if not self._ensure_workspace(inp): + return False + if inp.numel() > self.max_numel: + return False + return self._workspace.supports(inp) + + def pcie_ipc_all_reduce(self, inp: torch.Tensor) -> Optional[torch.Tensor]: + """All-reduce ``inp``, rebinding the workspace when the stream changes. + + One workspace serves one stream: its epoch and arrival counters assume the + calls sharing it are totally ordered. SGLang switches streams at phase + boundaries -- graph capture, then replay -- so the binding is moved with + the caller rather than kept on whichever stream got there first. This is + safe because those phases do not overlap; it would not be safe if two + streams issued reductions concurrently on the same group. + """ + stream = torch.cuda.current_stream() + if self._bound_stream is None or stream != self._bound_stream: + self._workspace.rebind_stream() + self._bound_stream = stream + return self._workspace.all_reduce(inp) + + def destroy(self) -> None: + if self._workspace is not None: + self._workspace.destroy() + self._workspace = None + self.disabled = True diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 7676ef3676..37c6732744 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -485,6 +485,28 @@ class GroupCoordinator: self.ca_comm: Optional[Any] = None self.qr_comm: Optional[QuickAllReduce] = None + + self.pcie_ipc_comm: Optional[Any] = None + from sglang.srt.distributed.device_communicators.pcie_ipc_ar import ( + PcieIpcCommunicator, + eligible_group, + ) + + if eligible_group( + group_name=group_name, + world_size=self.world_size, + deterministic=self._deterministic_collectives_enabled(), + ): + try: + # The IPC handshake needs the CUDA (NCCL) group, not the CPU one. + # Autotuning is the other way round: it rendezvouses on the host. + self.pcie_ipc_comm = PcieIpcCommunicator( + group=self.device_group, + device=self.device, + cpu_group=self.cpu_group, + ) + except Exception as e: + logger.warning(f"Setup FlashInfer PCIe-IPC allreduce failed with {e}.") if use_custom_allreduce and self.world_size > 1: # Initialize a custom fast all-reduce implementation. try: @@ -944,6 +966,16 @@ class GroupCoordinator: and self.ca_comm.should_custom_ar(input_) ): return "ca" + # After ``ca``: the PCIe-IPC kernels are for hosts where no fabric-specific + # backend applies. They do not probe for NVLink, so on a host that has it + # this ordering is what keeps the faster backend in front of them. + if ( + self.pcie_ipc_comm is not None + and not self.pcie_ipc_comm.disabled + and not should_use_pymscclpp_allreduce + and self.pcie_ipc_comm.should_pcie_ipc_ar(input_) + ): + return "pcie_ipc" if ( self.qr_comm is not None and not self.qr_comm.disabled @@ -1025,6 +1057,8 @@ class GroupCoordinator: elif outplace_all_reduce_method == "pymscclpp": assert not pymscclpp_comm.disabled out = pymscclpp_comm.all_reduce(input_) + elif outplace_all_reduce_method == "pcie_ipc": + return self.pcie_ipc_comm.pcie_ipc_all_reduce(input_) elif outplace_all_reduce_method == "pynccl": with pynccl_comm.change_state(enable=True): out = pynccl_comm.outplace_all_reduce(input_) @@ -2059,6 +2093,11 @@ class GroupCoordinator: return tensor def destroy(self): + # Must precede destroy_process_group(): FlashInfer's workspace + # teardown collectives on the group it was built with. + if self.pcie_ipc_comm is not None: + self.pcie_ipc_comm.destroy() + self.pcie_ipc_comm = None if self.device_group is not None: torch.distributed.destroy_process_group(self.device_group) self.device_group = None diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 6566413a75..787234e08e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1350,6 +1350,9 @@ class Envs: SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB = EnvInt(16 * 1024) SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PULL_SIZE_KB = EnvInt(None) SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PUSH_SIZE_KB = EnvInt(None) + # See docs/references/environment_variables. + SGLANG_ENABLE_PCIE_IPC_ALLREDUCE = EnvBool(False) + SGLANG_PCIE_IPC_MAX_NUMEL = EnvInt(0) # =================================================================== # RoPE cache diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index a3a3f97245..643b7d0b63 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -253,6 +253,7 @@ class BaseRunner(ABC): self._pre_initialize_flashinfer_allreduce_workspace() self._pre_initialize_fi_a2a_workspace() + self._pre_initialize_pcie_ipc_workspace() # Model-owned communication resources may depend on the resolved # request pool and must be compiled/allocated before graph capture. @@ -333,6 +334,44 @@ class BaseRunner(ABC): init_fi_a2a_workspace(get_parallel().dcp_group) + def _pre_initialize_pcie_ipc_workspace(self): + """Build the PCIe-IPC all-reduce workspace before graph capture. + + Runs for every model, not only the ones that autotune: left to the first + reduction, the build lands inside another autotune context and tuning + declines there. + """ + pcie_ipc_comm = get_parallel().tp_group.pcie_ipc_comm + if pcie_ipc_comm is None: + return + + mr = self.model_runner + pcie_ipc_comm.prepare( + hidden=mr.model_config.hidden_size, + max_rows=self._widest_decode_rows(), + ) + + def _widest_decode_rows(self) -> Optional[int]: + """Rows in the widest decode reduction, or None when no graph is captured. + + Same derivation the decode runner uses for its buffers, so the workspace + covers exactly the batches that will be issued: get_batch_sizes_to_capture + already applies the attention-tp alignment and the req_to_token_pool clamp, + which a second derivation from cuda_graph_config would miss. + """ + if get_exec().graph.cuda_graph_config is None: + return None + from sglang.srt.model_executor.runner.base_cuda_graph_runner import ( + get_batch_sizes_to_capture, + ) + + mr = self.model_runner + tokens_per_req = mr.decode_num_tokens_per_req() + capture_bs, _ = get_batch_sizes_to_capture(mr, tokens_per_req) + if not capture_bs: + return None + return max(capture_bs) * tokens_per_req + def _flashinfer_autotune(self, *, buffers, batch_size): """Run flashinfer autotune. diff --git a/test/registered/unit/distributed/test_pcie_ipc_ar.py b/test/registered/unit/distributed/test_pcie_ipc_ar.py new file mode 100644 index 0000000000..3c06bf8450 --- /dev/null +++ b/test/registered/unit/distributed/test_pcie_ipc_ar.py @@ -0,0 +1,513 @@ +"""Unit tests for the FlashInfer PCIe-IPC all-reduce adapter. + +These cover the parts that decide whether a reduction reaches the kernels at +all -- world-size gating, workspace sizing, and the shape guard -- because each +of them fails *silently* in production: an unsupported shape is not an error, +it is a fall back to NCCL, which looks exactly like the kernels being slow. +""" + +import unittest +from unittest.mock import MagicMock, patch + +import torch + +from sglang.srt.distributed.device_communicators import pcie_ipc_ar +from sglang.srt.distributed.device_communicators.pcie_ipc_ar import ( + PcieIpcCommunicator, +) +from sglang.srt.environ import envs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +HIDDEN = 6144 + + +def _make_comm( + world_size=8, workspace_cls=None, cpu_group=None, tune=None, max_rows=None +): + """Build a communicator with the collaborators stubbed out. + + The constructor is bypassed: it needs a live process group and a CUDA + device, neither of which this test has, and neither of which the logic + under test depends on. + + ``_ensure_workspace`` ends by calling ``_tune``, which imports FlashInfer + before it reads ``_cpu_group``. Neither belongs to the sizing or shape + logic these helpers serve, and the import fails outright on a CPU runner, + so tuning is stubbed by default and covered on its own in ``TestTuning``. + """ + comm = PcieIpcCommunicator.__new__(PcieIpcCommunicator) + comm.disabled = False + comm.max_numel = 0 + comm._workspace = None + comm._bound_stream = None + comm._group = MagicMock() + comm._device = torch.device("cpu") + comm._world_size = world_size + comm._workspace_cls = workspace_cls or MagicMock() + comm._build_failed = False + comm._cpu_group = cpu_group + comm._max_rows = max_rows + comm._tune = tune if tune is not None else (lambda hidden: None) + return comm + + +class TestWorldSizeGate(CustomTestCase): + def test_unsupported_world_size_disables(self): + """A world size the kernels have no IPC channels for must disable cleanly.""" + for world_size in (1, 3, 5, 6, 7, 16): + with self.subTest(world_size=world_size): + with patch.object( + pcie_ipc_ar.dist, "get_world_size", return_value=world_size + ): + comm = PcieIpcCommunicator(group=MagicMock(), device=0) + self.assertTrue(comm.disabled) + + def test_missing_flashinfer_disables_instead_of_raising(self): + """A build without pcie_ipc_comm must degrade to NCCL, not crash the server.""" + with ( + patch.object(pcie_ipc_ar.dist, "get_world_size", return_value=8), + patch("builtins.__import__", side_effect=ImportError("no pcie_ipc_comm")), + ): + comm = PcieIpcCommunicator(group=MagicMock(), device=0) + self.assertTrue(comm.disabled) + + +class TestGroupEligibility(CustomTestCase): + """Which groups get this backend, which is the rule that broke other models. + + ``"tp" in unique_name`` also matched attention_tp / moe_tp / + pdmux_prefill_tp. Those groups carry no pynccl or custom all-reduce + communicator, so a reduction dispatched to them hits an assertion instead + of falling back, and Qwen3-8B TP4/DP2 and Qwen3-30B-A3B TP4/EP2 fail to + start with the flag on. + """ + + def test_only_the_tensor_parallel_group_is_eligible(self): + with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True): + self.assertTrue( + pcie_ipc_ar.eligible_group( + group_name="tp", world_size=4, deterministic=False + ) + ) + for name in ("attention_tp", "moe_tp", "pdmux_prefill_tp", "world", "pp"): + with self.subTest(group=name): + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name=name, world_size=4, deterministic=False + ) + ) + + def test_deterministic_inference_keeps_the_backend_off(self): + """Deterministic inference forbids a shape-dependent reduction order. + + The kernels select a tactic per shape, so the order would follow the + batch shape -- which is why that mode already pins NCCL and turns off + custom and symmetric-memory all-reduce. Disabling tuning is not enough: + FlashInfer's seed policy is per-shape as well. + """ + with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True): + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + pcie_ipc_ar._warn_deterministic_wins.cache_clear() + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name="tp", world_size=4, deterministic=True + ) + ) + self.assertIn("Deterministic inference", "\n".join(logs.output)) + + def test_the_deterministic_predicate_still_exists(self): + """The caller supplies this flag, so a mock cannot catch a rename. + + GroupCoordinator owns the predicate -- it also folds in the + SGLANG_USE_1STAGE_ALLREDUCE override -- and this backend must keep + using that one rather than reading the env var a second time. + """ + from sglang.srt.distributed.parallel_state import GroupCoordinator + + self.assertTrue(hasattr(GroupCoordinator, "_deterministic_collectives_enabled")) + + def test_single_rank_group_is_not_eligible(self): + with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True): + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name="tp", world_size=1, deterministic=False + ) + ) + + def test_disabled_by_default(self): + """The backend is opt-in; nothing attaches without the flag.""" + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name="tp", world_size=4, deterministic=False + ) + ) + + def test_anonymous_group_is_not_eligible(self): + with envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.override(True): + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name=None, world_size=4, deterministic=False + ) + ) + self.assertFalse( + pcie_ipc_ar.eligible_group( + group_name="anonymous", world_size=4, deterministic=False + ) + ) + + +class TestWorkspaceRelease(CustomTestCase): + """The workspace must be released before the groups it was built on. + + FlashInfer collectives on that group while tearing the workspace down, so + ordering is the whole point: releasing afterwards is not a leak that shows + up in testing, it is a collective on a destroyed group. + """ + + def _coordinator(self, order): + from sglang.srt.distributed.parallel_state import GroupCoordinator + + coordinator = GroupCoordinator.__new__(GroupCoordinator) + coordinator.device_group = MagicMock(name="device_group") + coordinator.cpu_group = MagicMock(name="cpu_group") + coordinator.pynccl_comm = None + coordinator.pymscclpp_comm = None + coordinator.ca_comm = None + coordinator.mq_broadcaster = None + workspace = MagicMock() + workspace.destroy.side_effect = lambda: order.append("workspace") + coordinator.pcie_ipc_comm = workspace + return coordinator + + def test_released_before_the_process_groups(self): + order = [] + coordinator = self._coordinator(order) + with patch.object( + torch.distributed, + "destroy_process_group", + side_effect=lambda g: order.append("process_group"), + ): + coordinator.destroy() + + self.assertEqual(order[0], "workspace") + self.assertEqual(order.count("process_group"), 2) + self.assertIsNone(coordinator.pcie_ipc_comm) + + def test_destroy_without_the_backend_is_a_no_op(self): + order = [] + coordinator = self._coordinator(order) + coordinator.pcie_ipc_comm = None + with patch.object( + torch.distributed, + "destroy_process_group", + side_effect=lambda g: order.append("process_group"), + ): + coordinator.destroy() + self.assertEqual(order, ["process_group", "process_group"]) + + +class TestWorkspaceSizing(CustomTestCase): + def test_sized_for_decode_not_prefill(self): + """The default bound is the decode width, so prefill chunks stay on NCCL. + + Sizing for a prefill chunk was measured 66% worse on TTFT and bought + nothing on TPOT, so this is the behaviour that must not regress. + """ + comm = _make_comm() + with patch.object(pcie_ipc_ar, "_decode_width", return_value=64): + self.assertTrue( + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + ) + self.assertEqual(comm.max_numel, 64 * HIDDEN) + + prefill = torch.empty(16384, HIDDEN, dtype=torch.bfloat16) + self.assertGreater(prefill.numel(), comm.max_numel) + self.assertFalse(comm.should_pcie_ipc_ar(prefill)) + + def test_falls_back_when_server_args_absent(self): + """Embedded/unit use has no server args; the bound must still be finite.""" + comm = _make_comm() + with patch.object(pcie_ipc_ar, "_decode_width", return_value=None): + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + self.assertEqual(comm.max_numel, pcie_ipc_ar._FALLBACK_DECODE_WIDTH * HIDDEN) + + def test_env_override_wins(self): + comm = _make_comm() + with envs.SGLANG_PCIE_IPC_MAX_NUMEL.override(123456): + with patch.object(pcie_ipc_ar, "_decode_width", return_value=64): + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + self.assertEqual(comm.max_numel, 123456) + + def test_build_failure_disables_once(self): + """A workspace that cannot be allocated must not be retried every layer.""" + cls = MagicMock(side_effect=RuntimeError("out of IPC handles")) + comm = _make_comm(workspace_cls=cls) + with patch.object(pcie_ipc_ar, "_decode_width", return_value=64): + self.assertFalse( + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + ) + self.assertFalse( + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + ) + self.assertTrue(comm.disabled) + self.assertEqual(cls.call_count, 1) + + +class TestShapeGuard(CustomTestCase): + def _ready_comm(self): + comm = _make_comm() + with patch.object(pcie_ipc_ar, "_decode_width", return_value=64): + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + comm._workspace.supports.return_value = True + return comm + + def test_delegates_to_flashinfer_supports(self): + """Coverage is FlashInfer's decision; a rejected shape keeps the NCCL path.""" + comm = self._ready_comm() + inp = torch.empty(4, HIDDEN, dtype=torch.bfloat16) + self.assertTrue(comm.should_pcie_ipc_ar(inp)) + + comm._workspace.supports.return_value = False + self.assertFalse(comm.should_pcie_ipc_ar(inp)) + + def test_rejects_noncontiguous_and_1d(self): + comm = self._ready_comm() + self.assertFalse( + comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=torch.bfloat16).t()) + ) + self.assertFalse( + comm.should_pcie_ipc_ar(torch.empty(HIDDEN, dtype=torch.bfloat16)) + ) + + def test_rejects_dtypes_that_were_never_tuned(self): + """FlashInfer keys tuning results by dtype, so fp16 would run untuned. + + This adapter only ever tunes bf16, so anything else must keep its NCCL + path rather than run the seed policy while looking tuned. + """ + comm = self._ready_comm() + for dtype in (torch.float16, torch.float32): + with self.subTest(dtype=dtype): + self.assertFalse( + comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=dtype)) + ) + + def test_disabled_communicator_never_claims_a_tensor(self): + comm = self._ready_comm() + comm.disabled = True + self.assertFalse( + comm.should_pcie_ipc_ar(torch.empty(4, HIDDEN, dtype=torch.bfloat16)) + ) + + +class TestDecodeWidth(CustomTestCase): + """The width must come from the server's own decode config. + + Reading it from ``ServerArgs`` raised ``AttributeError`` on every call once + upstream moved the resolved config onto the exec bag, and a bare ``except`` + turned that into a permanent, silent fallback: whatever + ``--cuda-graph-max-bs-decode`` + the operator passed, the workspace was sized for a fixed 64 rows. + """ + + def _context(self, max_bs, algorithm=None, draft_tokens=None): + cg = MagicMock() + cg.decode.max_bs = max_bs + exec_bag = MagicMock() + exec_bag.graph.cuda_graph_config = cg + spec = MagicMock() + spec.speculative_algorithm = algorithm + spec.speculative_num_draft_tokens = draft_tokens + return { + "sglang.srt.runtime_context": MagicMock( + get_exec=lambda: exec_bag, get_spec=lambda: spec + ) + } + + def test_width_follows_the_resolved_decode_max_bs(self): + with patch.dict("sys.modules", self._context(max_bs=128)): + self.assertEqual(pcie_ipc_ar._decode_width(), 128) + + def test_width_multiplies_by_the_speculative_draft_width(self): + """max_bs counts requests; a verify pass carries several rows each.""" + with patch.dict( + "sys.modules", self._context(max_bs=64, algorithm="EAGLE", draft_tokens=4) + ): + self.assertEqual(pcie_ipc_ar._decode_width(), 256) + + def test_no_speculation_means_one_row_per_request(self): + with patch.dict( + "sys.modules", self._context(max_bs=64, algorithm=None, draft_tokens=4) + ): + self.assertEqual(pcie_ipc_ar._decode_width(), 64) + + def test_absent_config_is_reported_not_swallowed(self): + """The fallback is fine; being quiet about it is what hid this.""" + broken = MagicMock() + broken.graph.cuda_graph_config = None + with patch.dict( + "sys.modules", + {"sglang.srt.runtime_context": MagicMock(get_exec=lambda: broken)}, + ): + self.assertIsNone(pcie_ipc_ar._decode_width()) + + def test_the_attribute_path_it_reads_still_exists(self): + """Mocks cannot catch a field that upstream renamed or moved. + + The defect this class covers was exactly that: the derivation kept + reading a path that had stopped existing, and the tests around it were + all mocked, so nothing noticed. Pin the real chain. + """ + from sglang.srt import runtime_context + from sglang.srt.model_executor.cuda_graph_config import ( + default_cuda_graph_config, + ) + + self.assertTrue(hasattr(runtime_context, "get_exec")) + self.assertTrue(hasattr(runtime_context, "get_spec")) + self.assertTrue(hasattr(default_cuda_graph_config().decode, "max_bs")) + + def test_runner_width_wins_over_the_re_derivation(self): + """prepare() carries the width the decode graphs were captured for. + + get_batch_sizes_to_capture already applies the attention-tp alignment + and the req_to_token_pool clamp; _decode_width() cannot see either, so + when the runner supplies a width it has to take precedence. + """ + comm = _make_comm(max_rows=24) + with patch.object(pcie_ipc_ar, "_decode_width", return_value=512): + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + self.assertEqual(comm.max_numel, 24 * HIDDEN) + + def test_fallback_width_is_announced(self): + comm = _make_comm() + with patch.object(pcie_ipc_ar, "_decode_width", return_value=None): + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + comm._ensure_workspace(torch.empty(1, HIDDEN, dtype=torch.bfloat16)) + self.assertIn("fallback width", "\n".join(logs.output)) + self.assertEqual(comm.max_numel, pcie_ipc_ar._FALLBACK_DECODE_WIDTH * HIDDEN) + + +class TestTuning(CustomTestCase): + """Every way tuning declines leaves the kernels on FlashInfer's seed policy. + + ``tune()`` returning is not evidence it measured anything: it declines + shapes silently, and the three guards below return before calling it at + all. That is indistinguishable from a successful tune unless asserted, + which is how two earlier revisions of this adapter were misread. + """ + + def _comm(self, cpu_group=object(), tuned=("shape",)): + comm = _make_comm(cpu_group=cpu_group) + comm._workspace = MagicMock() + comm._workspace.tune.return_value = list(tuned) + comm.max_numel = 64 * HIDDEN + return comm + + @staticmethod + def _flashinfer(is_tuning_mode=False): + """Stand in for FlashInfer, which a CPU runner does not have installed.""" + autotuner = MagicMock() + autotuner.AutoTuner.get.return_value.is_tuning_mode = is_tuning_mode + return {"flashinfer": MagicMock(), "flashinfer.autotuner": autotuner} + + def _run_tune(self, comm, is_tuning_mode=False, capturing=False): + with ( + patch.dict("sys.modules", self._flashinfer(is_tuning_mode)), + patch.object( + torch.cuda, "is_current_stream_capturing", return_value=capturing + ), + ): + PcieIpcCommunicator._tune(comm, HIDDEN) + + def test_autotune_disabled_reads_the_real_flag(self): + """Exercise the helper itself, not a patched stand-in. + + Patching _autotune_disabled in the tests above cannot catch a wrong or + renamed field inside it -- the same blind spot that let _decode_width + read a path that had stopped existing. + """ + for flag in (True, False): + with self.subTest(flag=flag): + bag = MagicMock() + bag.kernel.disable_flashinfer_autotune = flag + with patch.dict( + "sys.modules", + {"sglang.srt.runtime_context": MagicMock(get_exec=lambda: bag)}, + ): + self.assertIs(pcie_ipc_ar._autotune_disabled(), flag) + + def test_the_flag_still_exists_on_the_exec_bag(self): + from sglang.srt.arg_groups.fields.exec_ import ExecKernel + + self.assertIn("disable_flashinfer_autotune", ExecKernel.__annotations__) + + def test_honours_disable_flashinfer_autotune(self): + """--disable-flashinfer-autotune must stop this tuning path too. + + The flag gates should_run_flashinfer_autotune, which gates prepare(). + The lazy build on the first reduction is a second way in, and it used to + tune and write a cache while the operator had asked for neither. + """ + comm = self._comm() + with patch.object(pcie_ipc_ar, "_autotune_disabled", return_value=True): + with self.assertLogs(pcie_ipc_ar.logger, level="INFO") as logs: + self._run_tune(comm) + self.assertIn("--disable-flashinfer-autotune", "\n".join(logs.output)) + comm._workspace.tune.assert_not_called() + + def test_declines_inside_another_autotune_context(self): + """FlashInfer will not profile a collective from a context it did not open.""" + comm = self._comm() + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + self._run_tune(comm, is_tuning_mode=True) + self.assertIn("another autotune context", "\n".join(logs.output)) + comm._workspace.tune.assert_not_called() + + def test_declines_without_a_host_group(self): + """The autotuner rendezvouses on the host; no CPU group means no measurement.""" + comm = self._comm(cpu_group=None) + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + self._run_tune(comm) + self.assertIn("no host group", "\n".join(logs.output)) + comm._workspace.tune.assert_not_called() + + def test_declines_under_graph_capture(self): + """Autotuning replays kernels, which cannot happen inside a capture.""" + comm = self._comm() + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + self._run_tune(comm, capturing=True) + self.assertIn("capture", "\n".join(logs.output).lower()) + comm._workspace.tune.assert_not_called() + + def test_warns_when_tune_covered_no_shapes(self): + """tune() declining every shape must not read as a successful tune.""" + comm = self._comm(tuned=()) + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING") as logs: + self._run_tune(comm) + comm._workspace.tune.assert_called_once() + self.assertIn("covered no shapes", "\n".join(logs.output)) + + def test_reports_the_shapes_it_measured(self): + comm = self._comm(tuned=("a", "b", "c")) + with self.assertLogs(pcie_ipc_ar.logger, level="INFO") as logs: + self._run_tune(comm) + comm._workspace.tune.assert_called_once() + _, kwargs = comm._workspace.tune.call_args + self.assertEqual(kwargs["dtype"], torch.bfloat16) + self.assertIs(kwargs["tune_group"], comm._cpu_group) + self.assertIn("autotuned 3 shape(s)", "\n".join(logs.output)) + + def test_tune_failure_keeps_the_seed_policy(self): + """A raising autotuner must not take the server down with it.""" + comm = self._comm() + comm._workspace.tune.side_effect = RuntimeError("nvlink probe failed") + with self.assertLogs(pcie_ipc_ar.logger, level="WARNING"): + self._run_tune(comm) + + +if __name__ == "__main__": + unittest.main()