[Feat] paged-experts: Serve MoE models whose weights do not fit VRAM (#42557)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Jan Schalkamp
2026-10-19 23:10:04 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 806e7423af
commit 8e7601ae4a
23 changed files with 2234 additions and 5 deletions
@@ -0,0 +1,212 @@
// Paged experts (srt/layers/moe/paged_experts): the residency decision and the expert page-in on
// the GPU, so a decode step pages experts without a host sync and can be captured in a CUDA graph.
//
// decide: for a step whose routed (token, expert) entries fit the K slots, keep every resident
// expert the step uses, give each missing expert the least recently used slot the step does not
// need, and write the page-in plan (src expert -> dst slot, count) to device buffers.
// gather: copy the planned experts from the pinned host store into their slots, for every paged
// tensor of a layer in one launch. The count is read on the device, so a replayed graph moves
// exactly the experts its decide chose.
#pragma once
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.h>
#include <sgl_kernel/utils.cuh>
#include <climits>
#include <cstdint>
#include <cuda_runtime.h>
namespace {
// One warp. The evictions depend on each other and are committed one at a time by lane 0; each
// victim search and the per-entry bookkeeping are spread over the warp.
__global__ void paged_experts_decide_kernel(
const int32_t* topk, // [T] routed expert ids of the step (negative: padding)
int T,
int E,
int K,
int32_t* step, // [1] step counter, advanced on the device so replays age the LRU
int32_t* slot_expert, // [K] expert in each slot (-1: empty)
int32_t* expert_slot, // [E] slot of each expert (-1: not resident)
int32_t* slot_lastuse, // [K] step each slot was last used
int32_t* src, // [K] out: experts to page in
int32_t* dst, // [K] out: their slots
int32_t* count) { // [1] out: number of page-ins
const int lane = threadIdx.x;
int now = 0;
if (lane == 0) now = ++(*step);
now = __shfl_sync(0xffffffff, now, 0);
for (int i = lane; i < T; i += 32) {
const int e = topk[i];
if (e < 0 || e >= E) continue;
const int s = expert_slot[e];
if (s >= 0) slot_lastuse[s] = now;
}
__syncwarp();
int n = 0;
for (int i = 0; i < T; ++i) {
const int e = topk[i];
if (e < 0 || e >= E || expert_slot[e] >= 0) continue; // the same for every lane
// Victim: the least recently used slot holding no expert of this step, lowest slot on
// ties. Every slot used this step carries `now`, so one is always left while T <= K.
int oldest = INT_MAX, victim = K;
for (int s = lane; s < K; s += 32) {
const int lastuse = slot_lastuse[s];
if (lastuse != now && lastuse < oldest) {
oldest = lastuse;
victim = s;
}
}
for (int offset = 16; offset > 0; offset >>= 1) {
const int other_oldest = __shfl_down_sync(0xffffffff, oldest, offset);
const int other_victim = __shfl_down_sync(0xffffffff, victim, offset);
if (other_oldest < oldest || (other_oldest == oldest && other_victim < victim)) {
oldest = other_oldest;
victim = other_victim;
}
}
victim = __shfl_sync(0xffffffff, victim, 0);
if (victim == K) continue;
if (lane == 0) {
const int old = slot_expert[victim];
if (old >= 0) expert_slot[old] = -1;
slot_expert[victim] = e;
expert_slot[e] = victim;
slot_lastuse[victim] = now;
src[n] = e;
dst[n] = victim;
}
++n;
__syncwarp(); // the next search reads this eviction
}
if (lane == 0) *count = n;
}
template <typename T>
__device__ void
copy_rows(const T* host, T* gpu, long row, int n, const int32_t* src, const int32_t* dst, long tid, long stride) {
for (long j = tid; j < n * row; j += stride) {
const long i = j / row, off = j % row;
gpu[dst[i] * row + off] = host[src[i] * row + off];
}
}
__global__ void paged_experts_gather_kernel(
const int64_t* hosts, // [P] UVA device pointer of each paged tensor's host store
const int64_t* gpus, // [P] each paged tensor's K-slot GPU table
const int64_t* words, // [P] 4-byte words per expert of each tensor
int P,
const int32_t* src,
const int32_t* dst,
const int32_t* count) {
const int n = *count;
const long stride = static_cast<long>(gridDim.x) * blockDim.x;
const long tid = static_cast<long>(blockIdx.x) * blockDim.x + threadIdx.x;
for (int p = 0; p < P; ++p) {
const long row = words[p];
if (row % 4 == 0) { // 16-byte rows: copy in float4, which the host link needs
copy_rows(
reinterpret_cast<const float4*>(hosts[p]),
reinterpret_cast<float4*>(gpus[p]),
row / 4,
n,
src,
dst,
tid,
stride);
} else { // small per-expert scalars
copy_rows(
reinterpret_cast<const int32_t*>(hosts[p]),
reinterpret_cast<int32_t*>(gpus[p]),
row,
n,
src,
dst,
tid,
stride);
}
}
}
void paged_experts_decide(
tvm::ffi::TensorView topk,
tvm::ffi::TensorView step,
tvm::ffi::TensorView slot_expert,
tvm::ffi::TensorView expert_slot,
tvm::ffi::TensorView slot_lastuse,
tvm::ffi::TensorView src,
tvm::ffi::TensorView dst,
tvm::ffi::TensorView count) {
using namespace sglang::host;
SymbolicSize E = {"num_experts"}, K = {"num_slots"}, T = {"num_entries"}, One = {"one"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({E}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(expert_slot);
TensorMatcher({K})
.with_dtype<int32_t>()
.with_device<kDLCUDA>(device_)
.verify(slot_expert)
.verify(slot_lastuse)
.verify(src)
.verify(dst);
TensorMatcher({T}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(topk);
TensorMatcher({One}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(step).verify(count);
RuntimeCheck(T.unwrap() <= K.unwrap(), "paged_experts_decide needs at most K routed entries, got ", T.unwrap());
LaunchKernel(1, 32, device_.unwrap())(
paged_experts_decide_kernel,
static_cast<const int32_t*>(topk.data_ptr()),
static_cast<int>(T.unwrap()),
static_cast<int>(E.unwrap()),
static_cast<int>(K.unwrap()),
static_cast<int32_t*>(step.data_ptr()),
static_cast<int32_t*>(slot_expert.data_ptr()),
static_cast<int32_t*>(expert_slot.data_ptr()),
static_cast<int32_t*>(slot_lastuse.data_ptr()),
static_cast<int32_t*>(src.data_ptr()),
static_cast<int32_t*>(dst.data_ptr()),
static_cast<int32_t*>(count.data_ptr()));
}
void paged_experts_gather(
tvm::ffi::TensorView hosts,
tvm::ffi::TensorView gpus,
tvm::ffi::TensorView words,
tvm::ffi::TensorView src,
tvm::ffi::TensorView dst,
tvm::ffi::TensorView count) {
using namespace sglang::host;
SymbolicSize P = {"num_tensors"}, K = {"num_slots"}, One = {"one"};
SymbolicDevice device_;
device_.set_options<kDLCUDA>();
TensorMatcher({P}).with_dtype<int64_t>().with_device<kDLCUDA>(device_).verify(hosts).verify(gpus).verify(words);
TensorMatcher({K}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(src).verify(dst);
TensorMatcher({One}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(count);
// Decode pages few experts per step, so launch cost matters more than link saturation: a
// 2048x512 grid saturates the link better but measured 5% slower end to end (Qwen3-30B, K=90).
// An empty plan exits at once.
LaunchKernel(256, 256, device_.unwrap())(
paged_experts_gather_kernel,
static_cast<const int64_t*>(hosts.data_ptr()),
static_cast<const int64_t*>(gpus.data_ptr()),
static_cast<const int64_t*>(words.data_ptr()),
static_cast<int>(P.unwrap()),
static_cast<const int32_t*>(src.data_ptr()),
static_cast<const int32_t*>(dst.data_ptr()),
static_cast<const int32_t*>(count.data_ptr()));
}
int64_t paged_experts_host_device_pointer(tvm::ffi::TensorView pinned) {
void* ptr = nullptr;
const cudaError_t err = cudaHostGetDevicePointer(&ptr, pinned.data_ptr(), 0);
sglang::host::RuntimeCheck(err == cudaSuccess, "cudaHostGetDevicePointer failed: ", cudaGetErrorString(err));
return reinterpret_cast<int64_t>(ptr);
}
} // namespace
@@ -0,0 +1,77 @@
"""Paged-experts residency kernels: the per-step decision and the expert page-in on the GPU.
Used by ``sglang.srt.layers.moe.paged_experts`` so a decode step pages experts without a host
sync and can be captured in a CUDA graph. All index tensors are int32 on the GPU.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, load_jit
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_module() -> Module:
return load_jit(
"paged_experts",
cuda_files=["moe/paged_experts.cuh"],
cuda_wrappers=[
("paged_experts_decide", "paged_experts_decide"),
("paged_experts_gather", "paged_experts_gather"),
(
"paged_experts_host_device_pointer",
"paged_experts_host_device_pointer",
),
],
)
def paged_experts_decide(
topk: torch.Tensor,
step: torch.Tensor,
slot_expert: torch.Tensor,
expert_slot: torch.Tensor,
slot_lastuse: torch.Tensor,
src: torch.Tensor,
dst: torch.Tensor,
count: torch.Tensor,
) -> None:
"""Make every expert in ``topk`` (``[T]`` routed ids, negative = padding, ``T <= K``) resident.
Keeps the resident ones, gives each missing one the least recently used slot the step does
not need, and updates the residency state in place: ``step`` ``[1]``, ``slot_expert`` and
``slot_lastuse`` ``[K]``, ``expert_slot`` ``[E]``. Writes the page-in plan to ``src``/``dst``
``[K]`` (expert -> slot) and its length to ``count`` ``[1]``.
"""
_jit_module().paged_experts_decide(
topk, step, slot_expert, expert_slot, slot_lastuse, src, dst, count
)
def paged_experts_gather(
hosts: torch.Tensor,
gpus: torch.Tensor,
words: torch.Tensor,
src: torch.Tensor,
dst: torch.Tensor,
count: torch.Tensor,
) -> None:
"""Copy experts ``src[i]`` into slots ``dst[i]`` for ``i < count[0]``, for every paged tensor.
``hosts``/``gpus``/``words`` are int64 GPU tensors with one entry per paged tensor: the UVA
device pointer of its pinned host store (``paged_experts_host_device_pointer``), the address of
its GPU table, and its size per expert in 4-byte words. Rows of a multiple of 16 bytes are
copied in 16-byte loads.
"""
_jit_module().paged_experts_gather(hosts, gpus, words, src, dst, count)
def paged_experts_host_device_pointer(pinned: torch.Tensor) -> int:
"""The address under which the GPU reads the pinned host tensor ``pinned``."""
return int(_jit_module().paged_experts_host_device_pointer(pinned))
@@ -955,6 +955,22 @@ class ExecMoe(msgspec.Struct):
"Enforce shared experts fusion even when it would normally be disabled (e.g. under DeepEP). Mutually exclusive with --disable-shared-experts-fusion.",
] = False
# -------------------------------------------------------------------------
# Paged experts
# -------------------------------------------------------------------------
enable_paged_experts: A[
bool,
"Keep only --paged-experts-num-resident routed experts per MoE layer on the GPU and page "
"the rest from pinned host memory on demand. Unquantized, block-FP8 or GPTQ experts, "
"single GPU. Decode CUDA graphs (full backend) are captured up to K // top_k requests; "
"prefill graphs must be breakable or disabled.",
] = False
paged_experts_num_resident: A[
Optional[int],
"Number of routed experts per MoE layer kept resident on the GPU. Required with "
"--enable-paged-experts; at least the router's top-k.",
] = None
# -------------------------------------------------------------------------
# Ktransformers/AMX expert parallelism
# -------------------------------------------------------------------------
@@ -178,6 +178,7 @@ def fused_marlin_moe(
gemm1_alpha: Optional[float] = None,
activation: str = "silu",
is_gated: bool = True,
no_combine: bool = False,
) -> torch.Tensor:
"""
This function computes a Mixture of Experts (MoE) layer using two sets of
@@ -202,6 +203,8 @@ def fused_marlin_moe(
- w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
- w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
- num_bits (int): The number of bits in expert weights quantization.
- no_combine (bool): Return the router-weighted per-(token, expert) outputs,
``[num_tokens, topk, hidden]``, instead of their top-k sum.
Returns:
- torch.Tensor: The output tensor after applying the MoE layer.
@@ -410,6 +413,9 @@ def fused_marlin_moe(
is_zp_float=False,
).view(-1, topk, K)
if no_combine:
return intermediate_cache3
output = zero_copy_context.get_moe_output(hidden_states)
if output is None:
output = hidden_states if inplace else torch.empty_like(hidden_states)
@@ -147,6 +147,12 @@ def _maybe_copy_weight_view_before_h2d(
return _copy_weight_view_before_h2d(loaded_weight)
def _weight_layout_method(layer):
"""The method whose weight layout the loaders follow: the method a wrapper loads for
(``weight_load_method``, set by paged experts) or the layer's own quant method."""
return getattr(layer, "weight_load_method", None) or layer.quant_method
def _get_deepep_comm_group(a2a_backend):
group = get_parallel().tp_group.device_group
@@ -470,6 +476,9 @@ class FusedMoE(torch.nn.Module):
)
self.quant_method = quant_method
# The method whose weight layout the loader follows when quant_method wraps another
# one (None: quant_method itself).
self.weight_load_method = None
server_args = get_server_args()
kt_config = create_kt_config_from_server_args(server_args, layer_id)
if kt_config is not None:
@@ -489,6 +498,12 @@ class FusedMoE(torch.nn.Module):
self.use_flashinfer_trtllm_moe,
self.use_deep_gemm,
)
if get_exec().moe.enable_paged_experts:
from sglang.srt.layers.moe.paged_experts import make_for_layer
self.quant_method = make_for_layer(
layer=self, base_method=self.quant_method
)
_validate_hpc_ops_quant_method(self.quant_method)
_validate_deepep_v2_quant_method(self.quant_method)
if (
@@ -754,7 +769,11 @@ class FusedMoE(torch.nn.Module):
# w1, gate_proj: Load into first logical weight of w13.
# w3, up_proj: Load into second logical weight of w13.
# trtllm cutlass kernel assumes differently
switch_w13 = getattr(self.quant_method, "load_up_proj_weight_first", False)
switch_w13 = getattr(
_weight_layout_method(self),
"load_up_proj_weight_first",
False,
)
if (
(switch_w13 and shard_id == "w1") or (not switch_w13 and shard_id == "w3")
) and self.moe_runner_config.is_gated:
@@ -1177,7 +1196,7 @@ class FusedMoE(torch.nn.Module):
# compressed-tensors checkpoints with packed weights are stored flipped
# TODO (mgoin): check self.quant_method.quant_config.quant_format
# against known CompressionFormat enum values that have this quality
method = self.quant_method
method = _weight_layout_method(self)
if self.scheme is not None:
method = self.scheme
if method.__class__.__name__ == "KTEPWrapperMethod":
@@ -1412,7 +1431,7 @@ class FusedMoE(torch.nn.Module):
) -> None:
# Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights
# into block layout; hot weight updates must restore canonical shapes first.
method = self.quant_method
method = _weight_layout_method(self)
if isinstance(method, KTEPWrapperMethod):
method = method.gpu_method
if isinstance(method, UnquantizedFusedMoEMethod):
@@ -1441,7 +1460,7 @@ class FusedMoE(torch.nn.Module):
# compressed-tensors checkpoints with packed weights are stored flipped
# TODO: check self.quant_method.quant_config.quant_format
# against known CompressionFormat enum values that have this quality
method = self.quant_method
method = _weight_layout_method(self)
if self.scheme is not None:
method = self.scheme
if isinstance(method, Fp8MoEMethod) and (
@@ -53,6 +53,9 @@ class MoeRunnerConfig:
apply_router_weight_on_input: bool = False
inplace: bool = True
no_combine: bool = False
# With no_combine: still apply the router weight, leaving only the top-k sum to
# the caller (triton and marlin runners).
no_combine_keep_router_weight: bool = False
routed_scaling_factor: Optional[float] = None
gemm1_alpha: Optional[float] = None
gemm1_beta: Optional[float] = None
@@ -149,6 +149,11 @@ def fused_experts_none_to_marlin(
f"Unsupported Marlin MoE activation: {runner_config.activation}"
)
# Marlin always applies the router weight inside its down GEMM.
assert (
not runner_config.no_combine or runner_config.no_combine_keep_router_weight
), "Marlin MoE supports no_combine only with no_combine_keep_router_weight"
# Use a per-call workspace so captured graphs cannot alias Marlin's
# inter-block reduction locks and deadlock during capture.
workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
@@ -205,6 +210,7 @@ def fused_experts_none_to_marlin(
gemm1_alpha=runner_config.gemm1_alpha,
activation=runner_config.activation,
is_gated=runner_config.is_gated,
no_combine=runner_config.no_combine,
).to(hidden_states.dtype)
return StandardCombineInput(
@@ -162,6 +162,7 @@ class TritonRunnerCore(MoeRunnerCore):
activation=self.config.activation,
is_gated=self.config.is_gated,
no_combine=self.config.no_combine,
no_combine_keep_router_weight=self.config.no_combine_keep_router_weight,
inplace=self.config.inplace,
apply_router_weight_on_input=self.config.apply_router_weight_on_input,
routed_scaling_factor=self.config.routed_scaling_factor,
@@ -262,6 +262,7 @@ def outplace_fused_experts(
gate_up_interleaved: bool = True,
a1_q: Optional[torch.Tensor] = None,
fuse_swiglu_interleaved: bool = False,
no_combine_keep_router_weight: bool = False,
) -> torch.Tensor:
return fused_experts_impl(
hidden_states,
@@ -296,6 +297,7 @@ def outplace_fused_experts(
gate_up_interleaved=gate_up_interleaved,
a1_q=a1_q,
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
no_combine_keep_router_weight=no_combine_keep_router_weight,
)
@@ -395,6 +397,7 @@ def fused_experts(
gate_up_interleaved=moe_runner_config.gate_up_interleaved,
a1_q=a1_q,
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
no_combine_keep_router_weight=moe_runner_config.no_combine_keep_router_weight,
)
@@ -556,6 +559,7 @@ def _fused_moe_kernel_sequence(
gate_up_interleaved: bool = True,
a1_q: Optional[torch.Tensor] = None,
fuse_swiglu_interleaved: bool = False,
no_combine_keep_router_weight: bool = False,
) -> torch.Tensor:
"""Run the MoE kernel/activation/kernel/combine sequence in a single shot.
@@ -911,7 +915,8 @@ def _fused_moe_kernel_sequence(
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input and not no_combine,
not apply_router_weight_on_input
and (not no_combine or no_combine_keep_router_weight),
1,
down_config or config,
compute_type=compute_type,
@@ -935,6 +940,30 @@ def _fused_moe_kernel_sequence(
del intermediate_cache2
return moe_combine_topk(
intermediate_cache3=intermediate_cache3,
out_hidden_states=out_hidden_states,
routed_scaling_factor=routed_scaling_factor,
no_combine=no_combine,
use_fused_moe_sum_all_reduce=use_fused_moe_sum_all_reduce,
out_slice=out_slice,
_use_intermediate=_use_intermediate,
)
def moe_combine_topk(
intermediate_cache3: torch.Tensor,
out_hidden_states: torch.Tensor,
routed_scaling_factor: Optional[float],
no_combine: bool = False,
use_fused_moe_sum_all_reduce: bool = False,
out_slice: Optional[torch.Tensor] = None,
_use_intermediate: bool = True,
) -> torch.Tensor:
"""Sum the routed-weighted ``[num_tokens, topk, hidden]`` expert outputs into
``out_hidden_states`` (the tail of ``_fused_moe_kernel_sequence``)."""
num_tokens, topk = intermediate_cache3.shape[:2]
if routed_scaling_factor is None:
routed_scaling_factor = 1.0
@@ -1057,6 +1086,7 @@ def fused_experts_impl(
gate_up_interleaved: bool = True,
a1_q: Optional[torch.Tensor] = None,
fuse_swiglu_interleaved: bool = False,
no_combine_keep_router_weight: bool = False,
):
padded_size = padding_size
if not (use_fp8_w8a8 or use_int8_w8a8) or block_shape is not None or _use_aiter:
@@ -1137,6 +1167,7 @@ def fused_experts_impl(
gate_up_interleaved=gate_up_interleaved,
a1_q=a1_q,
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
no_combine_keep_router_weight=no_combine_keep_router_weight,
)
@@ -0,0 +1,17 @@
"""Paged experts: serve an MoE model whose routed experts do not fit in GPU memory.
Each MoE layer keeps K of its E routed experts in a K-slot GPU table and pages the rest from
pinned host memory; outputs are bit-identical to the unpaged layer. See
``docs/docs/advanced_features/paged_experts.mdx`` for the design and the execution flow.
Modules: ``formats`` (per quantization method), ``runners`` (per runner backend), ``store``
(host copy and transfers), ``residency`` (slots and waves), ``executor`` (one forward step),
``method`` (the ``FusedMoEMethodBase`` wiring them into a ``FusedMoE`` layer).
"""
from sglang.srt.layers.moe.paged_experts.method import (
PagedExpertsMoEMethod,
make_for_layer,
)
__all__ = ["PagedExpertsMoEMethod", "make_for_layer"]
@@ -0,0 +1,137 @@
"""Runs a paged forward step: page in, the wave's GEMMs, merge and finish per the runner
contract.
``EagerExecutor`` decides residency on the host and serves any step, in as many waves as it
needs. ``DeviceExecutor`` decides and pages on the GPU without a host sync, so CUDA graphs can
capture it; it serves the steps whose routed entries fit the K slots (one wave), which covers
decode up to ``K // top_k`` requests. Both finish through the same contract.
"""
from __future__ import annotations
from typing import Optional
import torch
from sglang.srt.layers.moe.paged_experts.residency import (
DeviceResidency,
LRUPolicy,
plan_waves,
)
from sglang.srt.layers.moe.paged_experts.runners import RunnerContract, run_on_slots
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
class EagerExecutor:
def __init__(
self,
*,
base_method,
store: HostExpertStore,
policy: LRUPolicy,
contract: RunnerContract,
num_experts: int,
routed_scaling_factor: float,
device_residency: Optional[DeviceResidency] = None,
):
self.base_method = base_method
self.store = store
self.policy = policy
self.contract = contract
self.num_experts = num_experts
self.routed_scaling_factor = routed_scaling_factor
# Shared with a DeviceExecutor, whose steps (and graph replays) move it on the GPU.
self.device_residency = device_residency
def run(self, layer, dispatch_output):
if self.device_residency is not None:
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"Paged experts: a captured decode batch routes more entries than the K "
"resident slots; capture batch sizes must stay at or below K // top_k"
)
self.device_residency.load_into(self.policy)
out = self._run(layer, dispatch_output)
if self.device_residency is not None:
self.device_residency.store_from(self.policy)
return out
def _run(self, layer, dispatch_output):
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
topk_output = dispatch_output.topk_output
topk_ids = topk_output.topk_ids
distinct = [e for e in torch.unique(topk_ids).tolist() if e >= 0]
merged = None
waves = plan_waves(policy=self.policy, distinct=distinct)
for wave in waves:
self.store.page_in(layer=layer, loads=wave.loads)
# Index E (and -1, which wraps to it) stays -1: padded routing entries are masked.
to_slot = torch.full((self.num_experts + 1,), -1, dtype=torch.int32)
to_slot[wave.experts] = torch.tensor(wave.slots, dtype=torch.int32)
slots = to_slot.to(topk_ids.device)[topk_ids]
if len(waves) == 1: # only padding is outside the wave
merged = run_on_slots(self.base_method, layer, dispatch_output, slots)
break
merged = self.contract.run_wave(
base_method=self.base_method,
layer=layer,
dispatch_output=dispatch_output,
slots=slots,
merged=merged,
)
return StandardCombineInput(
hidden_states=self.contract.finish(
merged=merged,
hidden_states=dispatch_output.hidden_states,
routed_scaling_factor=self.routed_scaling_factor,
)
)
class DeviceExecutor:
"""One wave, decided and paged on the GPU: a step with at most K routed entries needs at
most K distinct experts, so they all fit the slots at once."""
def __init__(
self,
*,
base_method,
store: HostExpertStore,
residency: DeviceResidency,
contract: RunnerContract,
routed_scaling_factor: float,
):
self.base_method = base_method
self.store = store
self.residency = residency
self.contract = contract
self.routed_scaling_factor = routed_scaling_factor
def run(self, layer, dispatch_output):
from sglang.kernels.ops.moe.paged_experts import paged_experts_decide
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
r = self.residency
topk_ids = dispatch_output.topk_output.topk_ids
paged_experts_decide(
topk_ids.flatten().to(torch.int32),
r.step,
r.slot_expert,
r.expert_slot,
r.slot_lastuse,
r.src,
r.dst,
r.count,
)
self.store.gather(src=r.src, dst=r.dst, count=r.count)
# Every routed expert is now resident; padding (-1) stays masked.
slots = r.expert_slot[topk_ids.clamp(min=0)].masked_fill(topk_ids < 0, -1)
partial = run_on_slots(self.base_method, layer, dispatch_output, slots)
return StandardCombineInput(
hidden_states=self.contract.finish(
merged=partial,
hidden_states=dispatch_output.hidden_states,
routed_scaling_factor=self.routed_scaling_factor,
)
)
@@ -0,0 +1,184 @@
"""Per-quantization-method expert layout, one ``ExpertFormat`` per fused-MoE method.
A format answers three questions for the method it wraps:
* which per-expert tensors the checkpoint loads (``checkpoint_params``) and which are paged at
run time (``paged_params``);
* what happens after loading, so the host copy holds exactly what the GPU slots would hold
(for methods that repack their experts, the repack runs over the host copy K at a time);
* which runner contract (``runners.py``) its base method's runner follows.
Adding a quantization method means adding a subclass to ``FORMATS``.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Optional, Tuple
import torch
from sglang.srt.layers.moe.paged_experts.runners import (
MARLIN,
TRITON,
RunnerContract,
)
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
# Prefixes of FusedMoE's per-expert parameters.
_EXPERT_PARAM_PREFIXES = ("w13_", "w2_")
class ExpertFormat(ABC):
#: Per-expert tensors the base method creates for loading.
checkpoint_params: Tuple[str, ...] = ()
#: Per-expert tensors paged at run time.
paged_params: Tuple[str, ...] = ()
#: Whether the base post-load step repacks the experts; it then runs over the host copy K
#: experts at a time.
repacks_after_loading: bool = False
contract: RunnerContract = TRITON
@classmethod
@abstractmethod
def supports(cls, base_method) -> bool:
"""Whether this format handles ``base_method``."""
def check_params(self, layer, num_slots: int) -> None:
"""Raise unless the layer's per-expert tensors are exactly the declared ones."""
per_expert = {
name
for name, p in layer.named_parameters(recurse=False)
if name.startswith(_EXPERT_PARAM_PREFIXES)
and p.dim() > 0
and p.shape[0] == num_slots
}
if per_expert != set(self.checkpoint_params):
raise RuntimeError(
f"Paged experts: {type(self).__name__} expects per-expert parameters "
f"{sorted(self.checkpoint_params)}, the layer has {sorted(per_expert)}"
)
def after_loading(self, layer, base_method, store, num_slots: int):
"""Run the base post-load step; return the store to page from, whose experts 0..K-1
the GPU slots then hold."""
if self.repacks_after_loading:
return self._repack(
layer=layer,
base_method=base_method,
staged=store,
num_slots=num_slots,
)
for name, host in store.host.items():
getattr(layer, name).data.copy_(host[:num_slots])
base_method.process_weights_after_loading(layer)
# Paging copies host rows verbatim, so the base method must not transform the
# loaded weights (e.g. a kernel-specific shuffle or row interleave).
for name, host in store.host.items():
gpu = getattr(layer, name).data[0]
if not torch.equal(gpu, host[0].to(gpu.device)):
raise RuntimeError(
f"Paged experts: {type(base_method).__name__} transforms {name} "
"after loading, which the host store does not replicate"
)
return store
def _repack(self, layer, base_method, staged, num_slots: int):
"""Run the base post-load step over the staged checkpoint layout, K experts at a time,
collecting its per-expert output into a new store. Only valid for steps that treat
every expert independently."""
num_experts = next(iter(staged.host.values())).shape[0]
templates = {name: getattr(layer, name) for name in staged.host}
packed = None
for start in range(0, num_experts, num_slots):
n = min(num_slots, num_experts - start)
# Fresh parameters in the checkpoint layout: the step may replace or rewrite them.
for name, template in templates.items():
fresh = torch.nn.Parameter(
torch.empty_like(template.data), requires_grad=False
)
fresh.__dict__.update(template.__dict__)
fresh.data[:n].copy_(staged.host[name][start : start + n])
layer.register_parameter(name, fresh)
base_method.process_weights_after_loading(layer)
if packed is None:
packed = HostExpertStore(
layer=layer, names=self.paged_params, num_experts=num_experts
)
for name, host in packed.host.items():
host[start : start + n].copy_(getattr(layer, name).data[:n])
for name, host in packed.host.items():
getattr(layer, name).data.copy_(host[:num_slots])
return packed
class UnquantizedFormat(ExpertFormat):
checkpoint_params = paged_params = ("w13_weight", "w2_weight")
@classmethod
def supports(cls, base_method) -> bool:
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
return isinstance(base_method, UnquantizedFusedMoEMethod)
class Fp8BlockFormat(ExpertFormat):
"""FP8 weights with per-block scales (e.g. 128x128) and dynamic activation scales. On the
triton runner the post-load step leaves both untouched, so they page as stored."""
checkpoint_params = paged_params = (
"w13_weight",
"w2_weight",
"w13_weight_scale_inv",
"w2_weight_scale_inv",
)
@classmethod
def supports(cls, base_method) -> bool:
from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod
return (
isinstance(base_method, Fp8MoEMethod)
and base_method.quant_config.weight_block_size is not None
and base_method.quant_config.is_checkpoint_fp8_serialized
and not base_method.use_mxfp8
and not base_method.is_fp4_expert
)
class GptqMarlinFormat(ExpertFormat):
"""GPTQ int4/int8 experts on the Marlin runner. The post-load step sorts the act-order
indices and repacks each expert's weights and scales into Marlin's layout."""
checkpoint_params = (
"w13_qweight",
"w2_qweight",
"w13_scales",
"w2_scales",
"w13_qzeros",
"w2_qzeros",
"w13_g_idx",
"w2_g_idx",
"w13_g_idx_sort_indices",
"w2_g_idx_sort_indices",
)
# GPTQ-Marlin's MoE apply never passes the zero points to the kernel.
paged_params = tuple(n for n in checkpoint_params if not n.endswith("_qzeros"))
repacks_after_loading = True
contract = MARLIN
@classmethod
def supports(cls, base_method) -> bool:
from sglang.srt.layers.quantization.gptq.gptq import GPTQMarlinMoEMethod
return isinstance(base_method, GPTQMarlinMoEMethod)
FORMATS = (UnquantizedFormat, Fp8BlockFormat, GptqMarlinFormat)
def format_for(base_method) -> Optional[ExpertFormat]:
for fmt in FORMATS:
if fmt.supports(base_method):
return fmt()
return None
@@ -0,0 +1,323 @@
"""The fused-MoE method that wires paged experts into a ``FusedMoE`` layer."""
from __future__ import annotations
import logging
import torch
from sglang.srt.layers.moe.paged_experts.executor import DeviceExecutor, EagerExecutor
from sglang.srt.layers.moe.paged_experts.formats import ExpertFormat, format_for
from sglang.srt.layers.moe.paged_experts.residency import DeviceResidency, LRUPolicy
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.quantization.base_config import FusedMoEMethodBase
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
check_cuda_graph_backend,
cuda_graph_fully_disabled,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
eager_on_graph,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
is_in_breakable_cuda_graph,
)
from sglang.srt.runtime_context import (
get_exec,
get_model,
get_parallel,
get_spec,
process_model_config,
)
logger = logging.getLogger(__name__)
class PagedExpertsMoEMethod(FusedMoEMethodBase, torch.nn.Module):
"""Wraps a fused-MoE method so it runs over a K-slot GPU table paged from host memory.
An ``nn.Module`` like the methods it wraps, so it can replace one as a layer's child."""
def __init__(
self,
base_method,
expert_format: ExpertFormat,
num_experts: int,
num_resident: int,
graphs: bool = False,
):
torch.nn.Module.__init__(self)
self.base_method = base_method
self.format = expert_format
self.num_experts = num_experts
self.num_resident = num_resident
self.policy = LRUPolicy(num_resident)
self.store = None # built once the base method has created the expert tensors
self.executor = None # built once the store holds the paged layout
# Some steps may run inside a CUDA graph: decide one-wave steps on the device, so
# the graph can capture them.
self.graphs = graphs
self.device_executor = None
self.routed_scaling_factor = 1.0
def create_weights(
self,
layer,
num_experts,
hidden_size,
intermediate_size_per_partition,
params_dtype,
**extra,
):
# The GPU table has K rows. The loader below sends all E experts to the host store,
# so the native expert-id remap never sees ids >= K.
layer.num_local_experts = self.num_resident
layer._num_local_routed = layer._num_global_routed = self.num_resident
self.base_method.create_weights(
layer=layer,
num_experts=self.num_resident,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size_per_partition,
params_dtype=params_dtype,
**extra,
)
# The loader follows the base method's weight layout, as it would without paging.
layer.weight_load_method = self.base_method
layer._derived_weight_cache_error = (
"Online weight updates are not supported with --enable-paged-experts: "
"experts already on the GPU would keep the old weights."
)
self.format.check_params(layer=layer, num_slots=self.num_resident)
# A store that only stages the checkpoint layout for a repack need not be pinned.
self.store = HostExpertStore(
layer=layer,
names=self.format.checkpoint_params,
num_experts=self.num_experts,
pin=not self.format.repacks_after_loading,
)
for name in self.format.checkpoint_params:
getattr(layer, name).weight_loader = self._make_host_loader(
layer=layer, name=name
)
def _make_host_loader(self, layer, name):
def weight_loader(param, loaded_weight, weight_name, shard_id, expert_id):
# Reuse the native per-expert loader (sharding, w1/w3 placement, scale layout)
# on a one-row view of the host store, so host rows have exactly the native GPU
# layout. The row carries the parameter's loader tags (e.g. ``quant_method``).
row = torch.nn.Parameter(
self.store.host[name][expert_id : expert_id + 1], requires_grad=False
)
row.__dict__.update(param.__dict__)
layer._weight_loader_impl(
param=row,
loaded_weight=loaded_weight,
weight_name=weight_name,
shard_id=shard_id,
expert_id=0,
)
return weight_loader
def create_moe_runner(self, layer, moe_runner_config):
self.base_method.create_moe_runner(
layer=layer,
moe_runner_config=self.format.contract.runner_config(
moe_runner_config=moe_runner_config, num_slots=self.num_resident
),
)
self.runner = self.base_method.runner
self.format.contract.check(base_method=self.base_method)
self.routed_scaling_factor = moe_runner_config.routed_scaling_factor or 1.0
def process_weights_after_loading(self, layer):
self.store = self.format.after_loading(
layer=layer,
base_method=self.base_method,
store=self.store,
num_slots=self.num_resident,
)
residency = None
if self.graphs:
residency = DeviceResidency(
num_experts=self.num_experts,
num_slots=self.num_resident,
device=getattr(layer, self.format.paged_params[0]).device,
)
self.store.bind_device_gather(layer)
self.device_executor = DeviceExecutor(
base_method=self.base_method,
store=self.store,
residency=residency,
contract=self.format.contract,
routed_scaling_factor=self.routed_scaling_factor,
)
self.executor = EagerExecutor(
base_method=self.base_method,
store=self.store,
policy=self.policy,
contract=self.format.contract,
num_experts=self.num_experts,
routed_scaling_factor=self.routed_scaling_factor,
device_residency=residency,
)
def apply(self, layer, dispatch_output):
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
# Decided from shapes alone, so a captured graph always takes the same path.
topk_output = dispatch_output.topk_output
if (
self.device_executor is not None
and topk_output.topk_ids.numel() <= self.num_resident
):
return self.device_executor.run(
layer=layer, dispatch_output=dispatch_output
)
if is_in_breakable_cuda_graph():
# The host-planned waves run as an eager break between graph segments.
assert (
TopKOutputChecker.format_is_standard(topk_output)
and dispatch_output.hidden_states_scale is None
and dispatch_output.hidden_states_pre_quant is None
), (
"Paged experts: breakable CUDA graphs need standard, unquantized MoE inputs"
)
out = torch.empty_like(dispatch_output.hidden_states)
self._run_eager_into(
layer,
dispatch_output.hidden_states,
topk_output.topk_weights,
topk_output.topk_ids,
topk_output.router_logits,
out,
)
return StandardCombineInput(hidden_states=out)
return self.executor.run(layer=layer, dispatch_output=dispatch_output)
def _run_eager_into_impl(
self, layer, hidden_states, topk_weights, topk_ids, router_logits, out
) -> None:
# Flat tensors in, output buffer filled in place: a break replays with the tensors it
# was captured with, rebuilt into the dispatch output here.
from sglang.srt.layers.moe.token_dispatcher import StandardDispatchOutput
from sglang.srt.layers.moe.topk import StandardTopKOutput
dispatch_output = StandardDispatchOutput(
hidden_states=hidden_states,
hidden_states_scale=None,
topk_output=StandardTopKOutput(
topk_weights=topk_weights,
topk_ids=topk_ids,
router_logits=router_logits,
),
)
out.copy_(
self.executor.run(
layer=layer, dispatch_output=dispatch_output
).hidden_states
)
_run_eager_into = eager_on_graph(True)(_run_eager_into_impl)
def make_for_layer(layer, base_method) -> PagedExpertsMoEMethod:
expert_format = format_for(base_method)
check_paged_experts_compat(layer=layer, expert_format=expert_format)
num_experts = layer.num_local_experts
num_resident = min(get_exec().moe.paged_experts_num_resident, num_experts)
if check_cuda_graph_backend(Phase.DECODE, Backend.FULL):
# A full decode graph cannot break out for host-planned waves.
_cap_decode_capture_batch_sizes(max(1, num_resident // _router_top_k(layer)))
return PagedExpertsMoEMethod(
base_method=base_method,
expert_format=expert_format,
num_experts=num_experts,
num_resident=num_resident,
graphs=not cuda_graph_fully_disabled(),
)
def _cap_decode_capture_batch_sizes(max_bs: int) -> None:
"""Capture full decode graphs only at batch sizes whose routed entries fit the K slots,
the steps the device executor serves; larger batches run without a graph. The cap itself
is captured, so batches up to it pad to a graph. Every MoE layer calls this; only the
first changes the config."""
decode = get_exec().graph.cuda_graph_config.decode
if not decode.bs:
return
cap = min(max_bs, max(decode.bs))
capture_bs = sorted({bs for bs in decode.bs if bs <= cap} | {cap})
if capture_bs != list(decode.bs):
logger.info(
"Paged experts: decode CUDA graphs captured at batch sizes %s (at most K // top_k)",
capture_bs,
)
decode.bs = capture_bs
decode.max_bs = cap
def check_paged_experts_compat(layer, expert_format) -> None:
"""Raise if the server or layer configuration is outside what paged experts supports;
``expert_format`` is None when no format handles the layer's quantization method."""
problems = []
parallel = get_parallel()
for name in ("tp_size", "ep_size", "pp_size", "dp_size"):
if getattr(parallel, name) > 1:
problems.append(f"--{name.replace('_', '-')} must be 1 (single GPU only)")
if get_exec().moe.enable_eplb:
problems.append("--enable-eplb is not supported")
if get_spec().speculative_algorithm:
problems.append("speculative decoding is not supported")
# Host-planned waves run as eager breaks in a breakable prefill graph; a full graph only
# captures steps that fit one wave, which decode can be limited to and prefill cannot.
if not (
check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
or check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
):
problems.append(
"prefill CUDA graphs must use the breakable backend or be disabled"
)
if not (
check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
or check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED)
):
problems.append("decode CUDA graphs must use the full backend or be disabled")
if get_model().load_format == "dummy":
problems.append(
"--load-format dummy is not supported: the host store needs real weights"
)
num_resident = get_exec().moe.paged_experts_num_resident
top_k = _router_top_k(layer)
if num_resident is None:
problems.append("--paged-experts-num-resident is required")
elif num_resident < top_k:
problems.append(
f"--paged-experts-num-resident must be at least the router's top-k ({top_k})"
)
if expert_format is None:
problems.append(
"only unquantized (bf16/fp16), block-quantized FP8 and GPTQ experts are "
f"supported, got {type(layer.quant_method).__name__}"
)
if get_exec().moe.enable_fused_moe_sum_all_reduce:
problems.append(
"--enable-fused-moe-sum-all-reduce is not supported: paged experts sums the "
"top-k outputs itself"
)
if layer.num_fused_shared_experts:
problems.append(
"fused shared experts are not supported (--disable-shared-experts-fusion)"
)
if problems:
raise RuntimeError("Paged experts: " + "; ".join(problems))
def _router_top_k(layer) -> int:
# Models that route through a separate TopK module (e.g. OLMoE) leave FusedMoE's unset.
if layer.top_k is not None:
return layer.top_k
return process_model_config().hf_text_config.num_experts_per_tok
@@ -0,0 +1,99 @@
"""Which expert sits in which GPU slot, and how a step's experts are split into waves.
Pure bookkeeping: no expert moves here. The executor pages ``Wave.loads`` into place before it
runs the wave. With CUDA graphs, decode steps decide on the device (``DeviceResidency``), and
host-planned steps load that state into the policy and store it back.
"""
from __future__ import annotations
from collections import OrderedDict
from typing import List, Tuple
import msgspec
import torch
class Wave(msgspec.Struct, frozen=True):
experts: List[int] # logical expert ids served by this wave
slots: List[int] # the GPU slot of each expert, same order
loads: List[Tuple[int, int]] # (expert, slot) pairs to page in before the wave runs
class LRUPolicy:
"""Chooses the slot of each expert of a wave, evicting the least recently used experts
outside it. Slots start holding experts 0..K-1."""
def __init__(self, num_slots: int):
self.num_slots = num_slots
# Logical expert -> slot for the resident experts, least recently used first.
self.resident = OrderedDict((e, e) for e in range(num_slots))
def place(self, experts: List[int]) -> Wave:
"""Make ``experts`` (at most ``num_slots``) resident and return where they are."""
for e in experts:
if e in self.resident:
self.resident.move_to_end(e)
loads = []
for e in experts:
if e not in self.resident:
# Every expert of this wave sits at the LRU tail, so the head is never one of them.
_, slot = self.resident.popitem(last=False)
self.resident[e] = slot
loads.append((e, slot))
return Wave(
experts=experts, slots=[self.resident[e] for e in experts], loads=loads
)
def plan_waves(policy: LRUPolicy, distinct: List[int]) -> List[Wave]:
"""Split a step's distinct experts into waves of at most ``num_slots`` and place each.
A step with no routed expert (all padding) still gets one empty wave, so the layer runs.
"""
k = policy.num_slots
ordered = [e for e in distinct if e in policy.resident] + [
e for e in distinct if e not in policy.resident
]
groups = [ordered[i : i + k] for i in range(0, len(ordered), k)] or [[]]
return [policy.place(group) for group in groups]
class DeviceResidency:
"""The LRU residency state on the GPU, which the decide kernel updates in place, plus the
buffers of its page-in plan. Slots start holding experts 0..K-1, like ``LRUPolicy``."""
def __init__(self, num_experts: int, num_slots: int, device):
i32 = dict(dtype=torch.int32, device=device)
self.step = torch.zeros(1, **i32)
self.slot_expert = torch.arange(num_slots, **i32)
self.expert_slot = torch.full((num_experts,), -1, **i32)
self.expert_slot[:num_slots] = self.slot_expert
self.slot_lastuse = torch.zeros(num_slots, **i32)
self.src = torch.zeros(num_slots, **i32)
self.dst = torch.zeros(num_slots, **i32)
self.count = torch.zeros(1, **i32)
def load_into(self, policy: LRUPolicy) -> None:
"""Make ``policy`` hold this state: the same residents, least recently used first."""
experts = self.slot_expert.tolist()
lastuse = self.slot_lastuse.tolist()
order = sorted(range(len(experts)), key=lambda s: (lastuse[s], s))
policy.resident = OrderedDict((experts[s], s) for s in order)
def store_from(self, policy: LRUPolicy) -> None:
"""Make this state hold ``policy``'s, its recency order as the last steps."""
num_slots = self.slot_expert.numel()
base = int(self.step.item())
slot_expert = [0] * num_slots
lastuse = [0] * num_slots
for rank, (expert, slot) in enumerate(policy.resident.items(), start=1):
slot_expert[slot] = expert
lastuse[slot] = base + rank
self.slot_expert.copy_(torch.tensor(slot_expert, dtype=torch.int32))
self.slot_lastuse.copy_(torch.tensor(lastuse, dtype=torch.int32))
self.expert_slot.fill_(-1)
self.expert_slot[self.slot_expert.long()] = torch.arange(
num_slots, dtype=torch.int32, device=self.expert_slot.device
)
self.step.fill_(base + num_slots)
@@ -0,0 +1,146 @@
"""What paged experts needs from a fused-MoE runner backend, one ``RunnerContract`` per backend.
The executor runs a step in waves (one wave when its experts fit the K slots). The runner
returns every routed (token, expert) output already multiplied by its router weight
(``no_combine``); the contract runs each wave over only the entries whose expert it holds,
writes them into one merged buffer, and performs the top-k sum the runner would have performed
itself. A quantization format names the contract of the runner its base method uses; supporting
a new backend means adding a contract here.
"""
from __future__ import annotations
from dataclasses import replace
import torch
import torch.nn.functional as F
def run_on_slots(base_method, layer, dispatch_output, slots, outside: int = 0):
"""Run the base method over the K-slot table: routed entry i goes to GPU slot ``slots[i]``;
entries at -1 get expert id ``outside`` and router weight 0, so they contribute nothing."""
masked = slots < 0
topk_output = dispatch_output.topk_output
wave_output = dispatch_output._replace(
topk_output=topk_output._replace(
topk_ids=slots.masked_fill(masked, outside),
topk_weights=topk_output.topk_weights.masked_fill(masked, 0),
),
)
return base_method.apply(layer=layer, dispatch_output=wave_output).hidden_states
class RunnerContract:
def runner_config(self, moe_runner_config, num_slots: int):
"""The base runner's config: K local experts, router-weighted per-expert outputs."""
return replace(
moe_runner_config,
num_local_experts=num_slots,
inplace=False,
no_combine=True,
no_combine_keep_router_weight=True,
)
def check(self, base_method) -> None:
"""Raise if the base method's runner cannot honor ``runner_config``."""
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
"""Compute the routed entries of one wave (``slots`` >= 0) into ``merged``, the step's
``[tokens, top_k, hidden]`` entries (None before its first wave), and return it."""
raise NotImplementedError
def finish(self, merged, hidden_states, routed_scaling_factor: float):
"""The layer output: the top-k sum of the merged entries."""
out = torch.empty_like(hidden_states)
self._sum_top_k(merged, out, routed_scaling_factor)
return out
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
"""Sum ``[tokens, top_k, hidden]`` into ``out`` exactly as the runner would."""
raise NotImplementedError
class TritonContract(RunnerContract):
def check(self, base_method) -> None:
backend = base_method.runner.runner_backend
if not backend.is_triton():
raise RuntimeError(
f"Paged experts: requires the triton MoE runner, got {backend.value}"
)
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
config = base_method.runner.config
skips = (
config.num_experts is None or config.num_experts != config.num_local_experts
)
partial = run_on_slots(
base_method, layer, dispatch_output, slots, outside=-1 if skips else 0
)
if merged is None:
return partial
outside = (slots < 0).unsqueeze(-1)
return torch.where(outside, merged, partial, out=merged)
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
moe_combine_topk,
)
moe_combine_topk(
intermediate_cache3=entries,
out_hidden_states=out,
routed_scaling_factor=routed_scaling_factor,
)
class MarlinContract(RunnerContract):
# The Marlin MoE methods build a Marlin runner unconditionally, so there is nothing to check.
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
# Only the wave's entries, compacted: the runner sees no masked rows.
hidden_states = dispatch_output.hidden_states
assert (
dispatch_output.hidden_states_scale is None
and dispatch_output.hidden_states_pre_quant is None
), "Paged experts: compacted waves need unquantized MoE inputs"
if merged is None:
merged = hidden_states.new_zeros(*slots.shape, hidden_states.shape[-1])
entries = (slots >= 0).flatten().nonzero().squeeze(1)
num = entries.numel()
if num == 0:
return merged
# Padded to a few sizes per power of two, so the runner sees few distinct batch
# shapes; padding rows go to slot 0 with weight 0 and are dropped.
step = max(16, 1 << max(0, num.bit_length() - 4))
pad = -num % step
tokens = F.pad(entries // slots.shape[1], (0, pad))
topk = dispatch_output.topk_output
compact = dispatch_output._replace(
hidden_states=hidden_states[tokens],
topk_output=topk._replace(
topk_ids=F.pad(slots.flatten()[entries], (0, pad)).unsqueeze(1),
topk_weights=F.pad(
topk.topk_weights.flatten()[entries], (0, pad)
).unsqueeze(1),
router_logits=(
None if topk.router_logits is None else topk.router_logits[tokens]
),
),
)
out = base_method.apply(layer=layer, dispatch_output=compact).hidden_states
merged.view(-1, merged.shape[-1])[entries] = out.view(-1, merged.shape[-1])[
:num
]
return merged
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
# The reduce fused_marlin_moe itself ends with.
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import (
moe_sum_reduce,
)
moe_sum_reduce(entries, out, routed_scaling_factor)
TRITON = TritonContract()
MARLIN = MarlinContract()
@@ -0,0 +1,71 @@
"""The host copy of every expert and the transfer of experts into GPU slots."""
from __future__ import annotations
from typing import Dict, Sequence, Tuple
import torch
class HostExpertStore:
"""Holds all E experts of each paged tensor on the host, in the layer's GPU layout.
Page-locked (``pin``) when it serves paging; a store only staged for a post-load repack
stays pageable. ``page_in`` is planned on the host; ``gather`` reads a plan on the GPU.
The GPU tensors are looked up on the layer at every transfer, because a quantization
method may rebind them after loading.
"""
def __init__(self, layer, names: Sequence[str], num_experts: int, pin: bool = True):
self.pin = pin
self.host: Dict[str, torch.Tensor] = {
name: self._alloc(
shape=(num_experts, *getattr(layer, name).shape[1:]),
dtype=getattr(layer, name).dtype,
)
for name in names
}
def _alloc(self, shape, dtype: torch.dtype) -> torch.Tensor:
return torch.empty(shape, dtype=dtype, device="cpu", pin_memory=self.pin)
def page_in(self, layer, loads: Sequence[Tuple[int, int]]) -> None:
"""Copy each ``(expert, slot)`` pair's expert, of every paged tensor, into its slot."""
for name, host in self.host.items():
gpu = getattr(layer, name).data
for expert, slot in loads:
gpu[slot].copy_(host[expert], non_blocking=True)
def bind_device_gather(self, layer) -> None:
"""Prepare ``gather``; called once the layer's tensors are final. The gather kernel
reads the pinned store directly, through its UVA address."""
from sglang.kernels.ops.moe.paged_experts import (
paged_experts_host_device_pointer,
)
hosts, gpus, words = [], [], []
for name, host in self.host.items():
gpu = getattr(layer, name).data
row_bytes = host[0].numel() * host.element_size()
if row_bytes == 0:
continue # e.g. GPTQ without act-order keeps empty g_idx tensors
if not (self.pin and gpu.is_contiguous() and row_bytes % 4 == 0):
raise RuntimeError(
f"Paged experts: CUDA graphs need pinned, contiguous expert rows of whole "
f"4-byte words; {name} has {row_bytes} bytes per expert, "
f"pinned={host.is_pinned()}, contiguous={gpu.is_contiguous()}"
)
hosts.append(paged_experts_host_device_pointer(host))
gpus.append(gpu.data_ptr())
words.append(row_bytes // 4)
self._gather_args = [
torch.tensor(v, dtype=torch.int64, device=gpu.device)
for v in (hosts, gpus, words)
]
def gather(self, src: torch.Tensor, dst: torch.Tensor, count: torch.Tensor) -> None:
"""Copy expert ``src[i]`` into GPU slot ``dst[i]`` for ``i < count[0]``, with every
index read on the GPU (a device-planned step): capturable in a CUDA graph."""
from sglang.kernels.ops.moe.paged_experts import paged_experts_gather
paged_experts_gather(*self._gather_args, src, dst, count)
+5
View File
@@ -850,6 +850,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
self.alt_stream is not None
and get_is_capture_mode()
and not torch.compiler.is_compiling()
# Paged experts run as an eager break, which must end the segment on the
# capture stream.
and not (
get_exec().moe.enable_paged_experts and is_in_breakable_cuda_graph()
)
):
final_hidden_states, shared_output = self.forward_normal_dual_stream(
hidden_states,