[SM120] Add optional FlashInfer PCIe-IPC all-reduce for switch-free hosts (#34528)
This commit is contained in:
@@ -0,0 +1,426 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""FlashInfer PCIe-IPC all-reduce for switch-free intra-node machines.
|
||||
|
||||
FlashInfer's ``pcie_ipc_comm`` targets hosts where every peer transfer crosses the
|
||||
CPU root complex -- no NVLink, no multicast. The existing custom all-reduce
|
||||
backends assume one of those fabrics, so on such a host every per-layer reduction
|
||||
falls back to NCCL, which leaves a large margin on the table at decode sizes.
|
||||
|
||||
Why the kernel wins here
|
||||
------------------------
|
||||
Naive all-to-all peer writes collapse on this fabric, and the collapse worsens
|
||||
with the number of concurrently writing blocks. FlashInfer's kernels instead
|
||||
*stage* their pushes so each rank has exactly one outbound and one inbound stream
|
||||
at any instant, and the 8-rank path keeps a 4+4 island decomposition so the
|
||||
scarce cross-socket link carries the minimum. Measured on one 8x SM120 host
|
||||
(GPUs 0-3 and 4-7 on separate NUMA nodes), against NCCL, hidden 6144 bf16:
|
||||
|
||||
========= =========== ========== =========
|
||||
Message FlashInfer NCCL Speed-up
|
||||
========= =========== ========== =========
|
||||
12 KiB 5.3 us 205.1 us 38.8x
|
||||
96 KiB 15.3 us 584.9 us 38.2x
|
||||
384 KiB 34.6 us 1169.3 us 33.8x
|
||||
100 MiB 8.5 ms 50.4 ms 5.9x
|
||||
========= =========== ========== =========
|
||||
|
||||
:meth:`PcieIpcAllReduceWorkspace.supports` reports whether the kernels can run
|
||||
a shape, not whether they beat NCCL on it, so the workspace bound below is what
|
||||
keeps the reductions this backend should not serve on NCCL.
|
||||
|
||||
Workspace sizing
|
||||
----------------
|
||||
The workspace is sized for the widest decode reduction the runner will issue,
|
||||
``max_bs * draft_tokens * hidden``. The bound is a row count, not a phase
|
||||
split: a prefill chunk narrower than it is admitted too. What the bound rules
|
||||
out is the full-width prefill chunk, which is orders of magnitude larger, and
|
||||
there NCCL's ring is the better algorithm. Measured at 8 ranks, TP8, 8k
|
||||
context, against NCCL as the baseline:
|
||||
|
||||
=================== ========== ========== ===============
|
||||
Workspace TTFT TPOT Output tok/s
|
||||
=================== ========== ========== ===============
|
||||
NCCL only 1910 ms 21.14 ms 35.02
|
||||
prefill-sized 3176 ms 13.64 ms 38.43
|
||||
decode-sized 1849 ms 13.62 ms 48.05
|
||||
=================== ========== ========== ===============
|
||||
|
||||
Handing prefill to these kernels costs 66% on TTFT and buys nothing on TPOT.
|
||||
The decode-sized workspace keeps the whole TPOT win, stays level with NCCL on
|
||||
TTFT, and is the only one of the three that is not worse than NCCL anywhere we
|
||||
measured. It is also ~250x smaller, which matters at long context: at 128k with
|
||||
4 concurrent requests the prefill-sized workspace regressed TPOT by 45%, and
|
||||
that regression disappears when it is sized for decode.
|
||||
|
||||
The bound comes from the resolved decode config, but the hidden size is not
|
||||
known when the group is built, so the workspace is created on the first
|
||||
eligible tensor, whose trailing dimension is exactly that hidden size. Ranks
|
||||
run the same sequence of reductions, so they all reach that first call with the
|
||||
same shape and build the same workspace. ``SGLANG_PCIE_IPC_MAX_NUMEL`` overrides
|
||||
the derivation for deployments that want to hand larger reductions to the
|
||||
kernels anyway.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The kernels build IPC channels for these world sizes only.
|
||||
_SUPPORTED_WORLD_SIZES = (2, 4, 8)
|
||||
|
||||
# Fallback decode width when the server args are absent (unit tests, embedded
|
||||
# use). Covers the batch sizes we serve; anything larger stays on NCCL.
|
||||
_FALLBACK_DECODE_WIDTH = 64
|
||||
|
||||
|
||||
#: attention_tp / moe_tp / pdmux_prefill_tp carry no pynccl or custom
|
||||
#: all-reduce communicator, so a reduction dispatched to them asserts.
|
||||
_ELIGIBLE_GROUP_NAMES = frozenset({"tp"})
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _warn_deterministic_wins() -> None:
|
||||
logger.warning(
|
||||
"Deterministic inference is on; leaving PCIe-IPC all-reduce off. Its "
|
||||
"kernels pick a tactic per shape, so the reduction order would follow "
|
||||
"the batch shape."
|
||||
)
|
||||
|
||||
|
||||
def eligible_group(
|
||||
*, group_name: Optional[str], world_size: int, deterministic: bool
|
||||
) -> bool:
|
||||
"""Whether ``GroupCoordinator`` should build this backend for a group."""
|
||||
if not (
|
||||
envs.SGLANG_ENABLE_PCIE_IPC_ALLREDUCE.get()
|
||||
and world_size > 1
|
||||
and group_name in _ELIGIBLE_GROUP_NAMES
|
||||
):
|
||||
return False
|
||||
# Tactic is chosen per shape, so the reduction order would follow the
|
||||
# batch shape; disabling tuning alone does not help, the seed policy is
|
||||
# per-shape too.
|
||||
if deterministic:
|
||||
_warn_deterministic_wins()
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def _warn_untuned_dtype(dtype: torch.dtype) -> None:
|
||||
"""Once per dtype: a per-layer warning would be issued thousands of times."""
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC all-reduce serves bf16 only; %s reductions stay on NCCL.",
|
||||
dtype,
|
||||
)
|
||||
|
||||
|
||||
def _autotune_disabled() -> bool:
|
||||
"""Whether the operator asked for no FlashInfer autotuning."""
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
|
||||
try:
|
||||
return bool(get_exec().kernel.disable_flashinfer_autotune)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _decode_width() -> Optional[int]:
|
||||
"""Rows in the widest decode reduction, or None when the config is unavailable.
|
||||
|
||||
Decode runs inside a captured graph, so the largest captured batch bounds
|
||||
the reduction. ``max_bs`` counts requests, not rows: a speculative forward
|
||||
verifies several draft tokens per sequence, so the row count is
|
||||
``max_bs * draft tokens``, as in
|
||||
``runtime_context.cutedsl_moe_max_num_tokens``.
|
||||
"""
|
||||
from sglang.srt.runtime_context import get_exec, get_spec
|
||||
|
||||
try:
|
||||
cg_config = get_exec().graph.cuda_graph_config
|
||||
spec = get_spec()
|
||||
except ValueError:
|
||||
# Embedded and unit-test use have no runtime context; the caller then
|
||||
# sizes from a fixed width that ignores the server args, so say so.
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC has no runtime context to read the decode "
|
||||
"width from; falling back to a fixed bound."
|
||||
)
|
||||
return None
|
||||
if cg_config is None or not cg_config.decode.max_bs:
|
||||
return None
|
||||
rows_per_req = (
|
||||
(spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1
|
||||
)
|
||||
return cg_config.decode.max_bs * rows_per_req
|
||||
|
||||
|
||||
def _tune_cache_path(world_size: int) -> Optional[str]:
|
||||
"""Where FlashInfer should persist the measured tactics.
|
||||
|
||||
Named explicitly rather than left to default, because the tuning run has to
|
||||
write the file for the *next* server to reuse it; a workspace built without
|
||||
a cache path measures every start.
|
||||
"""
|
||||
try:
|
||||
from flashinfer.comm.pcie_ipc_tuning import default_cache_path
|
||||
|
||||
return str(default_cache_path(world_size))
|
||||
except (AttributeError, ImportError):
|
||||
return None
|
||||
|
||||
|
||||
def _tune_batches(max_rows: int) -> tuple[int, ...]:
|
||||
"""Row counts to profile, bounded by what the workspace can actually take.
|
||||
|
||||
The autotuner measures one tactic per shape, so profiling rows the workspace
|
||||
would reject is wasted startup time. Powers of two cover the captured decode
|
||||
batches; ``max_rows`` is appended because it is the widest reduction served
|
||||
and is not always a power of two.
|
||||
"""
|
||||
rows = [r for r in (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) if r <= max_rows]
|
||||
if max_rows >= 1 and max_rows not in rows:
|
||||
rows.append(max_rows)
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
class PcieIpcCommunicator:
|
||||
"""Adapter between ``GroupCoordinator`` and FlashInfer's PCIe-IPC all-reduce."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group: ProcessGroup,
|
||||
device: torch.device | int,
|
||||
cpu_group: Optional[ProcessGroup] = None,
|
||||
):
|
||||
self.disabled = True
|
||||
self.max_numel = 0
|
||||
self._workspace: Optional[Any] = None
|
||||
self._bound_stream: Optional[torch.cuda.Stream] = None
|
||||
self._cpu_group = cpu_group
|
||||
self._max_rows: Optional[int] = None
|
||||
|
||||
world_size = dist.get_world_size(group=group)
|
||||
if world_size not in _SUPPORTED_WORLD_SIZES:
|
||||
logger.debug(
|
||||
"FlashInfer PCIe-IPC all-reduce skipped: unsupported world size %d",
|
||||
world_size,
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(device, int):
|
||||
device = torch.device(f"cuda:{device}")
|
||||
if device.type != "cuda":
|
||||
return
|
||||
|
||||
try:
|
||||
from flashinfer.comm import PcieIpcAllReduceWorkspace
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"SGLANG_ENABLE_PCIE_IPC_ALLREDUCE is set but this FlashInfer build has "
|
||||
"no pcie_ipc_comm; falling back to NCCL all-reduce."
|
||||
)
|
||||
return
|
||||
|
||||
# The workspace is built on the first eligible tensor: sizing it needs the
|
||||
# hidden size, which the group does not know yet.
|
||||
self._group = group
|
||||
self._device = device
|
||||
self._world_size = world_size
|
||||
self._workspace_cls = PcieIpcAllReduceWorkspace
|
||||
self._build_failed = False
|
||||
self.disabled = False
|
||||
logger.info(
|
||||
"FlashInfer PCIe-IPC all-reduce enabled (world=%d, workspace sized on "
|
||||
"first reduction)",
|
||||
world_size,
|
||||
)
|
||||
|
||||
def _ensure_workspace(self, inp: torch.Tensor) -> bool:
|
||||
"""Build the workspace for the widest decode, using ``inp`` for the hidden size.
|
||||
|
||||
Every rank issues the same reductions in the same order, so they all arrive
|
||||
here with the same shape and agree on ``max_numel`` without extra exchange.
|
||||
"""
|
||||
if self._workspace is not None:
|
||||
return True
|
||||
if self._build_failed:
|
||||
return False
|
||||
|
||||
hidden = inp.shape[-1]
|
||||
override = envs.SGLANG_PCIE_IPC_MAX_NUMEL.get()
|
||||
if override:
|
||||
max_numel = override
|
||||
else:
|
||||
# The runner's width is already clamped to the request pool;
|
||||
# _decode_width() can only re-derive it.
|
||||
width = self._max_rows if self._max_rows else _decode_width()
|
||||
if width is None:
|
||||
# Admits a different set of batches than the server issues;
|
||||
# every reduction outside it silently uses NCCL.
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC is sizing its workspace from the fixed "
|
||||
"fallback width %d, not from this server's decode config; "
|
||||
"set SGLANG_PCIE_IPC_MAX_NUMEL to override.",
|
||||
_FALLBACK_DECODE_WIDTH,
|
||||
)
|
||||
width = _FALLBACK_DECODE_WIDTH
|
||||
max_numel = width * hidden
|
||||
|
||||
try:
|
||||
self._workspace = self._workspace_cls(
|
||||
group=self._group,
|
||||
max_numel=max_numel,
|
||||
dtype=torch.bfloat16,
|
||||
tune_batches=_tune_batches(max_numel // hidden),
|
||||
tune_cache=_tune_cache_path(self._world_size),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC workspace (max_numel=%d) failed: %s; "
|
||||
"reductions stay on NCCL",
|
||||
max_numel,
|
||||
e,
|
||||
)
|
||||
self._build_failed = True
|
||||
self.disabled = True
|
||||
return False
|
||||
|
||||
self.max_numel = max_numel
|
||||
logger.info(
|
||||
"FlashInfer PCIe-IPC workspace built (world=%d, max_numel=%d, hidden=%d)",
|
||||
self._world_size,
|
||||
max_numel,
|
||||
hidden,
|
||||
)
|
||||
self._tune(hidden)
|
||||
return True
|
||||
|
||||
def prepare(self, *, hidden: int, max_rows: Optional[int] = None) -> None:
|
||||
"""Build and measure ahead of the first reduction.
|
||||
|
||||
Tuning runs here rather than on the first reduction, which may land
|
||||
inside graph capture or another autotune pass; ``_tune`` declines in
|
||||
both, leaving the kernels on FlashInfer's seed policy.
|
||||
|
||||
``max_rows`` is the widest decode reduction the runner will issue.
|
||||
"""
|
||||
if self.disabled or self._workspace is not None:
|
||||
return
|
||||
self._max_rows = max_rows
|
||||
probe = torch.empty((1, hidden), dtype=torch.bfloat16, device=self._device)
|
||||
self._ensure_workspace(probe)
|
||||
|
||||
def _tune(self, hidden: int) -> None:
|
||||
"""Measure the launch tactics for this workspace's shapes.
|
||||
|
||||
Without this the kernels run FlashInfer's seed policy, and ``supports``
|
||||
answers from that policy rather than from measurements on this host.
|
||||
Results persist to FlashInfer's cache, so only the first server pays.
|
||||
"""
|
||||
if _autotune_disabled():
|
||||
logger.info(
|
||||
"FlashInfer PCIe-IPC all-reduce: --disable-flashinfer-autotune is "
|
||||
"set; keeping FlashInfer's seed policy."
|
||||
)
|
||||
return
|
||||
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
if AutoTuner.get().is_tuning_mode:
|
||||
# Reached through the lazy build, so tuning would start at
|
||||
# whatever point the first reduction happens; keep it to prepare().
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
|
||||
"another autotune context; skipping autotune and keeping the seed "
|
||||
"policy. Call PcieIpcCommunicator.prepare() from warmup to tune."
|
||||
)
|
||||
return
|
||||
if self._cpu_group is None:
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC all-reduce has no host group to autotune on; "
|
||||
"running FlashInfer's seed policy instead of measured tactics."
|
||||
)
|
||||
return
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC all-reduce reached its first reduction inside "
|
||||
"a graph capture; skipping autotune and keeping the seed policy."
|
||||
)
|
||||
return
|
||||
try:
|
||||
tuned = self._workspace.tune(
|
||||
[hidden], dtype=torch.bfloat16, tune_group=self._cpu_group
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC autotune failed (%s); keeping the seed policy.",
|
||||
e,
|
||||
)
|
||||
return
|
||||
# Report what tune() covered, not that it returned: it declines shapes
|
||||
# silently, and a log line that only proves no exception was raised is
|
||||
# how an autotune that measured nothing reads as a success.
|
||||
if not tuned:
|
||||
logger.warning(
|
||||
"FlashInfer PCIe-IPC autotune covered no shapes for hidden=%d "
|
||||
"(requested rows %s); the kernels keep the seed policy.",
|
||||
hidden,
|
||||
_tune_batches(self.max_numel // hidden),
|
||||
)
|
||||
return
|
||||
logger.info(
|
||||
"FlashInfer PCIe-IPC all-reduce autotuned %d shape(s) (hidden=%d, rows=%s)",
|
||||
len(tuned),
|
||||
hidden,
|
||||
_tune_batches(self.max_numel // hidden),
|
||||
)
|
||||
|
||||
def should_pcie_ipc_ar(self, inp: torch.Tensor) -> bool:
|
||||
"""Whether this reduction should go to the kernels rather than NCCL.
|
||||
|
||||
``supports`` answers whether the kernels can run the tensor, not
|
||||
whether they beat NCCL on it, so the size bound above is what keeps
|
||||
prefill-sized reductions on NCCL.
|
||||
"""
|
||||
if self.disabled or not inp.is_contiguous() or inp.dim() < 2:
|
||||
return False
|
||||
# FlashInfer keys tuning results by dtype, and only bf16 is tuned here.
|
||||
if inp.dtype is not torch.bfloat16:
|
||||
_warn_untuned_dtype(inp.dtype)
|
||||
return False
|
||||
if not self._ensure_workspace(inp):
|
||||
return False
|
||||
if inp.numel() > self.max_numel:
|
||||
return False
|
||||
return self._workspace.supports(inp)
|
||||
|
||||
def pcie_ipc_all_reduce(self, inp: torch.Tensor) -> Optional[torch.Tensor]:
|
||||
"""All-reduce ``inp``, rebinding the workspace when the stream changes.
|
||||
|
||||
One workspace serves one stream: its epoch and arrival counters assume the
|
||||
calls sharing it are totally ordered. SGLang switches streams at phase
|
||||
boundaries -- graph capture, then replay -- so the binding is moved with
|
||||
the caller rather than kept on whichever stream got there first. This is
|
||||
safe because those phases do not overlap; it would not be safe if two
|
||||
streams issued reductions concurrently on the same group.
|
||||
"""
|
||||
stream = torch.cuda.current_stream()
|
||||
if self._bound_stream is None or stream != self._bound_stream:
|
||||
self._workspace.rebind_stream()
|
||||
self._bound_stream = stream
|
||||
return self._workspace.all_reduce(inp)
|
||||
|
||||
def destroy(self) -> None:
|
||||
if self._workspace is not None:
|
||||
self._workspace.destroy()
|
||||
self._workspace = None
|
||||
self.disabled = True
|
||||
@@ -485,6 +485,28 @@ class GroupCoordinator:
|
||||
|
||||
self.ca_comm: Optional[Any] = None
|
||||
self.qr_comm: Optional[QuickAllReduce] = None
|
||||
|
||||
self.pcie_ipc_comm: Optional[Any] = None
|
||||
from sglang.srt.distributed.device_communicators.pcie_ipc_ar import (
|
||||
PcieIpcCommunicator,
|
||||
eligible_group,
|
||||
)
|
||||
|
||||
if eligible_group(
|
||||
group_name=group_name,
|
||||
world_size=self.world_size,
|
||||
deterministic=self._deterministic_collectives_enabled(),
|
||||
):
|
||||
try:
|
||||
# The IPC handshake needs the CUDA (NCCL) group, not the CPU one.
|
||||
# Autotuning is the other way round: it rendezvouses on the host.
|
||||
self.pcie_ipc_comm = PcieIpcCommunicator(
|
||||
group=self.device_group,
|
||||
device=self.device,
|
||||
cpu_group=self.cpu_group,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Setup FlashInfer PCIe-IPC allreduce failed with {e}.")
|
||||
if use_custom_allreduce and self.world_size > 1:
|
||||
# Initialize a custom fast all-reduce implementation.
|
||||
try:
|
||||
@@ -944,6 +966,16 @@ class GroupCoordinator:
|
||||
and self.ca_comm.should_custom_ar(input_)
|
||||
):
|
||||
return "ca"
|
||||
# After ``ca``: the PCIe-IPC kernels are for hosts where no fabric-specific
|
||||
# backend applies. They do not probe for NVLink, so on a host that has it
|
||||
# this ordering is what keeps the faster backend in front of them.
|
||||
if (
|
||||
self.pcie_ipc_comm is not None
|
||||
and not self.pcie_ipc_comm.disabled
|
||||
and not should_use_pymscclpp_allreduce
|
||||
and self.pcie_ipc_comm.should_pcie_ipc_ar(input_)
|
||||
):
|
||||
return "pcie_ipc"
|
||||
if (
|
||||
self.qr_comm is not None
|
||||
and not self.qr_comm.disabled
|
||||
@@ -1025,6 +1057,8 @@ class GroupCoordinator:
|
||||
elif outplace_all_reduce_method == "pymscclpp":
|
||||
assert not pymscclpp_comm.disabled
|
||||
out = pymscclpp_comm.all_reduce(input_)
|
||||
elif outplace_all_reduce_method == "pcie_ipc":
|
||||
return self.pcie_ipc_comm.pcie_ipc_all_reduce(input_)
|
||||
elif outplace_all_reduce_method == "pynccl":
|
||||
with pynccl_comm.change_state(enable=True):
|
||||
out = pynccl_comm.outplace_all_reduce(input_)
|
||||
@@ -2059,6 +2093,11 @@ class GroupCoordinator:
|
||||
return tensor
|
||||
|
||||
def destroy(self):
|
||||
# Must precede destroy_process_group(): FlashInfer's workspace
|
||||
# teardown collectives on the group it was built with.
|
||||
if self.pcie_ipc_comm is not None:
|
||||
self.pcie_ipc_comm.destroy()
|
||||
self.pcie_ipc_comm = None
|
||||
if self.device_group is not None:
|
||||
torch.distributed.destroy_process_group(self.device_group)
|
||||
self.device_group = None
|
||||
|
||||
@@ -1350,6 +1350,9 @@ class Envs:
|
||||
SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB = EnvInt(16 * 1024)
|
||||
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PULL_SIZE_KB = EnvInt(None)
|
||||
SGLANG_FORCE_CUSTOM_ALL_REDUCE_V2_PUSH_SIZE_KB = EnvInt(None)
|
||||
# See docs/references/environment_variables.
|
||||
SGLANG_ENABLE_PCIE_IPC_ALLREDUCE = EnvBool(False)
|
||||
SGLANG_PCIE_IPC_MAX_NUMEL = EnvInt(0)
|
||||
|
||||
# ===================================================================
|
||||
# RoPE cache
|
||||
|
||||
@@ -253,6 +253,7 @@ class BaseRunner(ABC):
|
||||
|
||||
self._pre_initialize_flashinfer_allreduce_workspace()
|
||||
self._pre_initialize_fi_a2a_workspace()
|
||||
self._pre_initialize_pcie_ipc_workspace()
|
||||
|
||||
# Model-owned communication resources may depend on the resolved
|
||||
# request pool and must be compiled/allocated before graph capture.
|
||||
@@ -333,6 +334,44 @@ class BaseRunner(ABC):
|
||||
|
||||
init_fi_a2a_workspace(get_parallel().dcp_group)
|
||||
|
||||
def _pre_initialize_pcie_ipc_workspace(self):
|
||||
"""Build the PCIe-IPC all-reduce workspace before graph capture.
|
||||
|
||||
Runs for every model, not only the ones that autotune: left to the first
|
||||
reduction, the build lands inside another autotune context and tuning
|
||||
declines there.
|
||||
"""
|
||||
pcie_ipc_comm = get_parallel().tp_group.pcie_ipc_comm
|
||||
if pcie_ipc_comm is None:
|
||||
return
|
||||
|
||||
mr = self.model_runner
|
||||
pcie_ipc_comm.prepare(
|
||||
hidden=mr.model_config.hidden_size,
|
||||
max_rows=self._widest_decode_rows(),
|
||||
)
|
||||
|
||||
def _widest_decode_rows(self) -> Optional[int]:
|
||||
"""Rows in the widest decode reduction, or None when no graph is captured.
|
||||
|
||||
Same derivation the decode runner uses for its buffers, so the workspace
|
||||
covers exactly the batches that will be issued: get_batch_sizes_to_capture
|
||||
already applies the attention-tp alignment and the req_to_token_pool clamp,
|
||||
which a second derivation from cuda_graph_config would miss.
|
||||
"""
|
||||
if get_exec().graph.cuda_graph_config is None:
|
||||
return None
|
||||
from sglang.srt.model_executor.runner.base_cuda_graph_runner import (
|
||||
get_batch_sizes_to_capture,
|
||||
)
|
||||
|
||||
mr = self.model_runner
|
||||
tokens_per_req = mr.decode_num_tokens_per_req()
|
||||
capture_bs, _ = get_batch_sizes_to_capture(mr, tokens_per_req)
|
||||
if not capture_bs:
|
||||
return None
|
||||
return max(capture_bs) * tokens_per_req
|
||||
|
||||
def _flashinfer_autotune(self, *, buffers, batch_size):
|
||||
"""Run flashinfer autotune.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user