From 8e7601ae4a48c1127afd1bcdccf0a0ad61dbf043 Mon Sep 17 00:00:00 2001 From: Jan Schalkamp Date: Mon, 19 Oct 2026 17:10:04 +0200 Subject: [PATCH] [Feat] paged-experts: Serve MoE models whose weights do not fit VRAM (#42557) Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- docs/docs.json | 1 + docs/docs/advanced_features/paged_experts.mdx | 138 +++++++ .../advanced_features/server_arguments.mdx | 32 ++ .../kernels/jit/csrc/moe/paged_experts.cuh | 212 ++++++++++ .../sglang/kernels/ops/moe/paged_experts.py | 77 ++++ python/sglang/srt/arg_groups/fields/exec_.py | 16 + .../moe/fused_moe_triton/fused_marlin_moe.py | 6 + .../srt/layers/moe/fused_moe_triton/layer.py | 27 +- .../sglang/srt/layers/moe/moe_runner/base.py | 3 + .../srt/layers/moe/moe_runner/marlin.py | 6 + .../srt/layers/moe/moe_runner/triton.py | 1 + .../moe/moe_runner/triton_utils/fused_moe.py | 33 +- .../srt/layers/moe/paged_experts/__init__.py | 17 + .../srt/layers/moe/paged_experts/executor.py | 137 +++++++ .../srt/layers/moe/paged_experts/formats.py | 184 +++++++++ .../srt/layers/moe/paged_experts/method.py | 323 +++++++++++++++ .../srt/layers/moe/paged_experts/residency.py | 99 +++++ .../srt/layers/moe/paged_experts/runners.py | 146 +++++++ .../srt/layers/moe/paged_experts/store.py | 71 ++++ python/sglang/srt/models/qwen2_moe.py | 5 + test/registered/e2e/moe/test_paged_experts.py | 166 ++++++++ .../ops/moe/test_paged_experts_kernels.py | 170 ++++++++ .../moe/paged_experts/test_paged_experts.py | 369 ++++++++++++++++++ 23 files changed, 2234 insertions(+), 5 deletions(-) create mode 100644 docs/docs/advanced_features/paged_experts.mdx create mode 100644 python/sglang/kernels/jit/csrc/moe/paged_experts.cuh create mode 100644 python/sglang/kernels/ops/moe/paged_experts.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/__init__.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/executor.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/formats.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/method.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/residency.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/runners.py create mode 100644 python/sglang/srt/layers/moe/paged_experts/store.py create mode 100644 test/registered/e2e/moe/test_paged_experts.py create mode 100644 test/registered/kernels/ops/moe/test_paged_experts_kernels.py create mode 100644 test/registered/unit/layers/moe/paged_experts/test_paged_experts.py diff --git a/docs/docs.json b/docs/docs.json index 948cc9678a..ac24019d9f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -957,6 +957,7 @@ "docs/advanced_features/prefill_cp", "docs/advanced_features/dcp", "docs/advanced_features/expert_parallelism", + "docs/advanced_features/paged_experts", "docs/advanced_features/lora", "docs/advanced_features/pd_disaggregation", "docs/advanced_features/epd_disaggregation", diff --git a/docs/docs/advanced_features/paged_experts.mdx b/docs/docs/advanced_features/paged_experts.mdx new file mode 100644 index 0000000000..a3a32799cb --- /dev/null +++ b/docs/docs/advanced_features/paged_experts.mdx @@ -0,0 +1,138 @@ +--- +title: "Paged Experts" +metatags: + description: "SGLang Paged Experts: serve an MoE model whose experts do not fit in GPU memory by keeping K of E experts per layer on the GPU and paging the rest from pinned host memory. Computation stays on the GPU." +--- + +Paged Experts serves a Mixture-of-Experts (MoE) model whose expert weights do not fit in GPU +memory. Each MoE layer keeps **K of its E routed experts in a K-slot table on the GPU** and holds +all E experts in pinned host memory; every step, the experts the step routes to that are not +resident are copied into slots before the expert GEMM runs. + +## Why Paged Experts? + +Expert weights are most of an MoE model's parameters, yet each token is routed to only `top_k` +of the `E` experts per layer, and consecutive tokens reuse many of the same experts. Paged +Experts keeps a working set of K experts per layer resident and pages the rest in from host +memory as routing requires them, so expert capacity is bounded by host RAM rather than VRAM. + +**Computation stays on the GPU.** Only expert *storage* is host-backed: experts are copied to the +GPU and computed there by the model's own fused-MoE kernel. This differs from `--cpu-offload-gb` +and CPU expert offload, which move the computation to the CPU. + +**Results are unchanged.** Routing is untouched (E-wide), and the paged layer produces the same +output as the unpaged one: bit-identical for unquantized and block-FP8 experts (with +`--enable-deterministic-inference`), and within the Marlin kernel's own run-to-run noise for +GPTQ. See [Correctness](#correctness). + +## Usage + +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-30B-A3B-GPTQ-Int4 \ + --enable-paged-experts \ + --paged-experts-num-resident 92 +``` + +See [Choosing K](#choosing-k) for the value. The default CUDA graphs stay on: decode graphs are captured up to +`K // top_k` concurrent requests, and breakable prefill graphs run the paged MoE layers as eager +breaks. + +## Related Parameters + +- **`--enable-paged-experts`**: Enable Paged Experts. Off by default. +- **`--paged-experts-num-resident K`**: Routed experts per MoE layer kept on the GPU. Required + with `--enable-paged-experts`, at least the router's `top_k`; larger values page less but leave + less memory for the KV cache. + +## Supported Configurations and Limitations + +- **Quantization:** unquantized (bf16/fp16), block-quantized FP8 (e.g. `weight_block_size` + `[128, 128]`) on the triton MoE runner, and GPTQ int4/int8 on the Marlin runner. Other + quantization methods are rejected at startup. +- **Single GPU:** tensor, expert, pipeline and data parallelism, EPLB, fused shared experts + (use `--disable-shared-experts-fusion`), `--enable-fused-moe-sum-all-reduce` and + `--load-format dummy` are rejected at startup. +- **Speculative decoding** is rejected at startup. +- **CUDA graphs:** decode graphs may use the `full` backend (the default) or be disabled; + prefill graphs may use the `breakable` backend (the default on CUDA) or be disabled. Other + combinations are rejected at startup. +- **Host memory:** the host store holds all E experts of every MoE layer and is page-locked, so + the host must be able to pin that much memory. Under WSL2, pinned memory is capped at about + half of system RAM. +- **Startup:** for GPTQ, the Marlin repack runs over the host store K experts at a time at load. + +## How It Works + +### The K-slot table and the host store + +The paged method wraps the layer's own fused-MoE method. It creates the layer's expert tensors +with K rows instead of E, so the GPU holds only the K slots, and allocates a host store holding +all E experts of each per-expert tensor. Each tensor's weight loader is redirected to a one-row +view of the host store, so checkpoint weights are written by SGLang's native per-expert loader +(sharding, gate/up placement, scale layout) and the host rows have exactly the layout the GPU +slots would have. + +After loading, the wrapped method's post-load step runs as it would without paging. For methods +that repack their experts (GPTQ's Marlin repack), it runs over the host store K experts at a +time, so the host store holds every expert in the repacked layout. + +### A step: waves, page-in, merge + +Every forward step of every MoE layer: + +1. The distinct experts the step routes to are split into **waves** of at most K, the experts + already resident first. Decode at + moderate batch sizes needs one wave; a prefill chunk usually needs several. +2. Each wave makes its experts resident, evicting the least recently used experts outside the + wave, and copies the missing ones from the host store into their slots. +3. The routed expert ids are remapped to slots and the wrapped fused-MoE kernel runs over the + wave's own entries **without its final top-k sum**, returning each routed (token, expert) + output already multiplied by its router weight. +4. Each entry is written into one merged buffer by the one wave that holds its expert, and the + merged entries go through the same top-k sum the kernel would have performed, so the result + does not depend on how the experts were split into waves. + +### Captured decode + +With decode CUDA graphs on, a step whose routed entries fit the K slots (`tokens × top_k ≤ K`, +i.e. decode up to `K // top_k` requests) is served by a single wave decided and paged on the +GPU, with no host synchronization, so the decode graph captures it: + +- a JIT kernel makes the LRU decision on the device and writes the page-in plan (experts, slots, + count) to GPU buffers; +- a second kernel copies the planned experts of every paged tensor from the pinned host store + into their slots, reading the host memory directly through its unified address and the count + on the device, so each graph replay moves exactly the experts it decided on. + +Decode capture batch sizes are capped at `K // top_k` (logged at startup); larger batches run +without a graph. Prefill steps plan their waves on the host: under a breakable prefill graph the +paged MoE layer is an eager break between graph segments, so the rest of the layer stack stays +captured. Both paths share one LRU state, so residency carries over between captured and +uncaptured steps. + +## Choosing K + +The K-slot table counts as model weights in SGLang's memory accounting, so K and the KV cache +come out of the same static budget (`--mem-fraction-static`): whatever K does not take becomes KV +cache, as without paging. One resident expert per layer costs `per_expert_bytes × num_moe_layers` +of GPU memory, so K is roughly + +``` +K ≈ (free_memory × mem_fraction_static − non_expert_bytes − kv_cache_bytes) / (per_expert_bytes × num_moe_layers) +``` + +For example, Qwen3-30B-A3B-GPTQ-Int4 (2.43 MB per expert, 48 MoE layers, 117 MB per slot) fits +K = 92 of 128 on a 16 GB GPU with `--mem-fraction-static 0.88 --max-total-tokens 10240`. +If the server runs out of memory for the KV cache at startup, lower K; for more concurrency, +lower K or use `--kv-cache-dtype fp8`. + +## Correctness + +Prompt and greedy decode logprobs are compared with the unpaged server under +`--enable-deterministic-inference --attention-backend triton`, eager and with decode captured: + +- **unquantized and block-FP8 experts (triton runner):** bit-identical; +- **GPTQ (Marlin runner):** Marlin's reduction depends on the batch shape, so a paged step (which + runs the kernel on a different number of experts) differs from the unpaged one within Marlin's + own batch-to-batch noise. diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 2e6e8ed3c8..2f3c66d589 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -2211,6 +2211,38 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec +## Paged experts + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ArgumentDescriptionDefaultsOptions
`--enable-paged-experts`Keep only `--paged-experts-num-resident` routed experts per MoE layer on the GPU and page the rest from pinned host memory on demand. Unquantized, block-FP8 or GPTQ experts, single GPU. Decode CUDA graphs (full backend) are captured up to K // top_k requests; prefill graphs must be breakable or disabled.`False`bool flag (set to enable)
`--paged-experts-num-resident`Number of routed experts per MoE layer kept resident on the GPU. Required with `--enable-paged-experts`; at least the router's top-k.`None`Type: int
+ ## Ktransformers diff --git a/python/sglang/kernels/jit/csrc/moe/paged_experts.cuh b/python/sglang/kernels/jit/csrc/moe/paged_experts.cuh new file mode 100644 index 0000000000..c5c5d5adca --- /dev/null +++ b/python/sglang/kernels/jit/csrc/moe/paged_experts.cuh @@ -0,0 +1,212 @@ +// Paged experts (srt/layers/moe/paged_experts): the residency decision and the expert page-in on +// the GPU, so a decode step pages experts without a host sync and can be captured in a CUDA graph. +// +// decide: for a step whose routed (token, expert) entries fit the K slots, keep every resident +// expert the step uses, give each missing expert the least recently used slot the step does not +// need, and write the page-in plan (src expert -> dst slot, count) to device buffers. +// gather: copy the planned experts from the pinned host store into their slots, for every paged +// tensor of a layer in one launch. The count is read on the device, so a replayed graph moves +// exactly the experts its decide chose. + +#pragma once + +#include +#include + +#include + +#include +#include +#include + +namespace { + +// One warp. The evictions depend on each other and are committed one at a time by lane 0; each +// victim search and the per-entry bookkeeping are spread over the warp. +__global__ void paged_experts_decide_kernel( + const int32_t* topk, // [T] routed expert ids of the step (negative: padding) + int T, + int E, + int K, + int32_t* step, // [1] step counter, advanced on the device so replays age the LRU + int32_t* slot_expert, // [K] expert in each slot (-1: empty) + int32_t* expert_slot, // [E] slot of each expert (-1: not resident) + int32_t* slot_lastuse, // [K] step each slot was last used + int32_t* src, // [K] out: experts to page in + int32_t* dst, // [K] out: their slots + int32_t* count) { // [1] out: number of page-ins + const int lane = threadIdx.x; + int now = 0; + if (lane == 0) now = ++(*step); + now = __shfl_sync(0xffffffff, now, 0); + for (int i = lane; i < T; i += 32) { + const int e = topk[i]; + if (e < 0 || e >= E) continue; + const int s = expert_slot[e]; + if (s >= 0) slot_lastuse[s] = now; + } + __syncwarp(); + int n = 0; + for (int i = 0; i < T; ++i) { + const int e = topk[i]; + if (e < 0 || e >= E || expert_slot[e] >= 0) continue; // the same for every lane + // Victim: the least recently used slot holding no expert of this step, lowest slot on + // ties. Every slot used this step carries `now`, so one is always left while T <= K. + int oldest = INT_MAX, victim = K; + for (int s = lane; s < K; s += 32) { + const int lastuse = slot_lastuse[s]; + if (lastuse != now && lastuse < oldest) { + oldest = lastuse; + victim = s; + } + } + for (int offset = 16; offset > 0; offset >>= 1) { + const int other_oldest = __shfl_down_sync(0xffffffff, oldest, offset); + const int other_victim = __shfl_down_sync(0xffffffff, victim, offset); + if (other_oldest < oldest || (other_oldest == oldest && other_victim < victim)) { + oldest = other_oldest; + victim = other_victim; + } + } + victim = __shfl_sync(0xffffffff, victim, 0); + if (victim == K) continue; + if (lane == 0) { + const int old = slot_expert[victim]; + if (old >= 0) expert_slot[old] = -1; + slot_expert[victim] = e; + expert_slot[e] = victim; + slot_lastuse[victim] = now; + src[n] = e; + dst[n] = victim; + } + ++n; + __syncwarp(); // the next search reads this eviction + } + if (lane == 0) *count = n; +} + +template +__device__ void +copy_rows(const T* host, T* gpu, long row, int n, const int32_t* src, const int32_t* dst, long tid, long stride) { + for (long j = tid; j < n * row; j += stride) { + const long i = j / row, off = j % row; + gpu[dst[i] * row + off] = host[src[i] * row + off]; + } +} + +__global__ void paged_experts_gather_kernel( + const int64_t* hosts, // [P] UVA device pointer of each paged tensor's host store + const int64_t* gpus, // [P] each paged tensor's K-slot GPU table + const int64_t* words, // [P] 4-byte words per expert of each tensor + int P, + const int32_t* src, + const int32_t* dst, + const int32_t* count) { + const int n = *count; + const long stride = static_cast(gridDim.x) * blockDim.x; + const long tid = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + for (int p = 0; p < P; ++p) { + const long row = words[p]; + if (row % 4 == 0) { // 16-byte rows: copy in float4, which the host link needs + copy_rows( + reinterpret_cast(hosts[p]), + reinterpret_cast(gpus[p]), + row / 4, + n, + src, + dst, + tid, + stride); + } else { // small per-expert scalars + copy_rows( + reinterpret_cast(hosts[p]), + reinterpret_cast(gpus[p]), + row, + n, + src, + dst, + tid, + stride); + } + } +} + +void paged_experts_decide( + tvm::ffi::TensorView topk, + tvm::ffi::TensorView step, + tvm::ffi::TensorView slot_expert, + tvm::ffi::TensorView expert_slot, + tvm::ffi::TensorView slot_lastuse, + tvm::ffi::TensorView src, + tvm::ffi::TensorView dst, + tvm::ffi::TensorView count) { + using namespace sglang::host; + + SymbolicSize E = {"num_experts"}, K = {"num_slots"}, T = {"num_entries"}, One = {"one"}; + SymbolicDevice device_; + device_.set_options(); + TensorMatcher({E}).with_dtype().with_device(device_).verify(expert_slot); + TensorMatcher({K}) + .with_dtype() + .with_device(device_) + .verify(slot_expert) + .verify(slot_lastuse) + .verify(src) + .verify(dst); + TensorMatcher({T}).with_dtype().with_device(device_).verify(topk); + TensorMatcher({One}).with_dtype().with_device(device_).verify(step).verify(count); + RuntimeCheck(T.unwrap() <= K.unwrap(), "paged_experts_decide needs at most K routed entries, got ", T.unwrap()); + + LaunchKernel(1, 32, device_.unwrap())( + paged_experts_decide_kernel, + static_cast(topk.data_ptr()), + static_cast(T.unwrap()), + static_cast(E.unwrap()), + static_cast(K.unwrap()), + static_cast(step.data_ptr()), + static_cast(slot_expert.data_ptr()), + static_cast(expert_slot.data_ptr()), + static_cast(slot_lastuse.data_ptr()), + static_cast(src.data_ptr()), + static_cast(dst.data_ptr()), + static_cast(count.data_ptr())); +} + +void paged_experts_gather( + tvm::ffi::TensorView hosts, + tvm::ffi::TensorView gpus, + tvm::ffi::TensorView words, + tvm::ffi::TensorView src, + tvm::ffi::TensorView dst, + tvm::ffi::TensorView count) { + using namespace sglang::host; + + SymbolicSize P = {"num_tensors"}, K = {"num_slots"}, One = {"one"}; + SymbolicDevice device_; + device_.set_options(); + TensorMatcher({P}).with_dtype().with_device(device_).verify(hosts).verify(gpus).verify(words); + TensorMatcher({K}).with_dtype().with_device(device_).verify(src).verify(dst); + TensorMatcher({One}).with_dtype().with_device(device_).verify(count); + + // Decode pages few experts per step, so launch cost matters more than link saturation: a + // 2048x512 grid saturates the link better but measured 5% slower end to end (Qwen3-30B, K=90). + // An empty plan exits at once. + LaunchKernel(256, 256, device_.unwrap())( + paged_experts_gather_kernel, + static_cast(hosts.data_ptr()), + static_cast(gpus.data_ptr()), + static_cast(words.data_ptr()), + static_cast(P.unwrap()), + static_cast(src.data_ptr()), + static_cast(dst.data_ptr()), + static_cast(count.data_ptr())); +} + +int64_t paged_experts_host_device_pointer(tvm::ffi::TensorView pinned) { + void* ptr = nullptr; + const cudaError_t err = cudaHostGetDevicePointer(&ptr, pinned.data_ptr(), 0); + sglang::host::RuntimeCheck(err == cudaSuccess, "cudaHostGetDevicePointer failed: ", cudaGetErrorString(err)); + return reinterpret_cast(ptr); +} + +} // namespace diff --git a/python/sglang/kernels/ops/moe/paged_experts.py b/python/sglang/kernels/ops/moe/paged_experts.py new file mode 100644 index 0000000000..e2cc02f587 --- /dev/null +++ b/python/sglang/kernels/ops/moe/paged_experts.py @@ -0,0 +1,77 @@ +"""Paged-experts residency kernels: the per-step decision and the expert page-in on the GPU. + +Used by ``sglang.srt.layers.moe.paged_experts`` so a decode step pages experts without a host +sync and can be captured in a CUDA graph. All index tensors are int32 on the GPU. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.jit.utils import cache_once, load_jit + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_module() -> Module: + return load_jit( + "paged_experts", + cuda_files=["moe/paged_experts.cuh"], + cuda_wrappers=[ + ("paged_experts_decide", "paged_experts_decide"), + ("paged_experts_gather", "paged_experts_gather"), + ( + "paged_experts_host_device_pointer", + "paged_experts_host_device_pointer", + ), + ], + ) + + +def paged_experts_decide( + topk: torch.Tensor, + step: torch.Tensor, + slot_expert: torch.Tensor, + expert_slot: torch.Tensor, + slot_lastuse: torch.Tensor, + src: torch.Tensor, + dst: torch.Tensor, + count: torch.Tensor, +) -> None: + """Make every expert in ``topk`` (``[T]`` routed ids, negative = padding, ``T <= K``) resident. + + Keeps the resident ones, gives each missing one the least recently used slot the step does + not need, and updates the residency state in place: ``step`` ``[1]``, ``slot_expert`` and + ``slot_lastuse`` ``[K]``, ``expert_slot`` ``[E]``. Writes the page-in plan to ``src``/``dst`` + ``[K]`` (expert -> slot) and its length to ``count`` ``[1]``. + """ + _jit_module().paged_experts_decide( + topk, step, slot_expert, expert_slot, slot_lastuse, src, dst, count + ) + + +def paged_experts_gather( + hosts: torch.Tensor, + gpus: torch.Tensor, + words: torch.Tensor, + src: torch.Tensor, + dst: torch.Tensor, + count: torch.Tensor, +) -> None: + """Copy experts ``src[i]`` into slots ``dst[i]`` for ``i < count[0]``, for every paged tensor. + + ``hosts``/``gpus``/``words`` are int64 GPU tensors with one entry per paged tensor: the UVA + device pointer of its pinned host store (``paged_experts_host_device_pointer``), the address of + its GPU table, and its size per expert in 4-byte words. Rows of a multiple of 16 bytes are + copied in 16-byte loads. + """ + _jit_module().paged_experts_gather(hosts, gpus, words, src, dst, count) + + +def paged_experts_host_device_pointer(pinned: torch.Tensor) -> int: + """The address under which the GPU reads the pinned host tensor ``pinned``.""" + return int(_jit_module().paged_experts_host_device_pointer(pinned)) diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 88d30e677b..210b8c7b9e 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -955,6 +955,22 @@ class ExecMoe(msgspec.Struct): "Enforce shared experts fusion even when it would normally be disabled (e.g. under DeepEP). Mutually exclusive with --disable-shared-experts-fusion.", ] = False + # ------------------------------------------------------------------------- + # Paged experts + # ------------------------------------------------------------------------- + enable_paged_experts: A[ + bool, + "Keep only --paged-experts-num-resident routed experts per MoE layer on the GPU and page " + "the rest from pinned host memory on demand. Unquantized, block-FP8 or GPTQ experts, " + "single GPU. Decode CUDA graphs (full backend) are captured up to K // top_k requests; " + "prefill graphs must be breakable or disabled.", + ] = False + paged_experts_num_resident: A[ + Optional[int], + "Number of routed experts per MoE layer kept resident on the GPU. Required with " + "--enable-paged-experts; at least the router's top-k.", + ] = None + # ------------------------------------------------------------------------- # Ktransformers/AMX expert parallelism # ------------------------------------------------------------------------- diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py index 3244e7e089..a3617a5721 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py @@ -178,6 +178,7 @@ def fused_marlin_moe( gemm1_alpha: Optional[float] = None, activation: str = "silu", is_gated: bool = True, + no_combine: bool = False, ) -> torch.Tensor: """ This function computes a Mixture of Experts (MoE) layer using two sets of @@ -202,6 +203,8 @@ def fused_marlin_moe( - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1. - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2. - num_bits (int): The number of bits in expert weights quantization. + - no_combine (bool): Return the router-weighted per-(token, expert) outputs, + ``[num_tokens, topk, hidden]``, instead of their top-k sum. Returns: - torch.Tensor: The output tensor after applying the MoE layer. @@ -410,6 +413,9 @@ def fused_marlin_moe( is_zp_float=False, ).view(-1, topk, K) + if no_combine: + return intermediate_cache3 + output = zero_copy_context.get_moe_output(hidden_states) if output is None: output = hidden_states if inplace else torch.empty_like(hidden_states) diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index a2d9fb1292..d4ca349e73 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -147,6 +147,12 @@ def _maybe_copy_weight_view_before_h2d( return _copy_weight_view_before_h2d(loaded_weight) +def _weight_layout_method(layer): + """The method whose weight layout the loaders follow: the method a wrapper loads for + (``weight_load_method``, set by paged experts) or the layer's own quant method.""" + return getattr(layer, "weight_load_method", None) or layer.quant_method + + def _get_deepep_comm_group(a2a_backend): group = get_parallel().tp_group.device_group @@ -470,6 +476,9 @@ class FusedMoE(torch.nn.Module): ) self.quant_method = quant_method + # The method whose weight layout the loader follows when quant_method wraps another + # one (None: quant_method itself). + self.weight_load_method = None server_args = get_server_args() kt_config = create_kt_config_from_server_args(server_args, layer_id) if kt_config is not None: @@ -489,6 +498,12 @@ class FusedMoE(torch.nn.Module): self.use_flashinfer_trtllm_moe, self.use_deep_gemm, ) + if get_exec().moe.enable_paged_experts: + from sglang.srt.layers.moe.paged_experts import make_for_layer + + self.quant_method = make_for_layer( + layer=self, base_method=self.quant_method + ) _validate_hpc_ops_quant_method(self.quant_method) _validate_deepep_v2_quant_method(self.quant_method) if ( @@ -754,7 +769,11 @@ class FusedMoE(torch.nn.Module): # w1, gate_proj: Load into first logical weight of w13. # w3, up_proj: Load into second logical weight of w13. # trtllm cutlass kernel assumes differently - switch_w13 = getattr(self.quant_method, "load_up_proj_weight_first", False) + switch_w13 = getattr( + _weight_layout_method(self), + "load_up_proj_weight_first", + False, + ) if ( (switch_w13 and shard_id == "w1") or (not switch_w13 and shard_id == "w3") ) and self.moe_runner_config.is_gated: @@ -1177,7 +1196,7 @@ class FusedMoE(torch.nn.Module): # compressed-tensors checkpoints with packed weights are stored flipped # TODO (mgoin): check self.quant_method.quant_config.quant_format # against known CompressionFormat enum values that have this quality - method = self.quant_method + method = _weight_layout_method(self) if self.scheme is not None: method = self.scheme if method.__class__.__name__ == "KTEPWrapperMethod": @@ -1412,7 +1431,7 @@ class FusedMoE(torch.nn.Module): ) -> None: # Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights # into block layout; hot weight updates must restore canonical shapes first. - method = self.quant_method + method = _weight_layout_method(self) if isinstance(method, KTEPWrapperMethod): method = method.gpu_method if isinstance(method, UnquantizedFusedMoEMethod): @@ -1441,7 +1460,7 @@ class FusedMoE(torch.nn.Module): # compressed-tensors checkpoints with packed weights are stored flipped # TODO: check self.quant_method.quant_config.quant_format # against known CompressionFormat enum values that have this quality - method = self.quant_method + method = _weight_layout_method(self) if self.scheme is not None: method = self.scheme if isinstance(method, Fp8MoEMethod) and ( diff --git a/python/sglang/srt/layers/moe/moe_runner/base.py b/python/sglang/srt/layers/moe/moe_runner/base.py index 77adb35301..6b8ba0890d 100644 --- a/python/sglang/srt/layers/moe/moe_runner/base.py +++ b/python/sglang/srt/layers/moe/moe_runner/base.py @@ -53,6 +53,9 @@ class MoeRunnerConfig: apply_router_weight_on_input: bool = False inplace: bool = True no_combine: bool = False + # With no_combine: still apply the router weight, leaving only the top-k sum to + # the caller (triton and marlin runners). + no_combine_keep_router_weight: bool = False routed_scaling_factor: Optional[float] = None gemm1_alpha: Optional[float] = None gemm1_beta: Optional[float] = None diff --git a/python/sglang/srt/layers/moe/moe_runner/marlin.py b/python/sglang/srt/layers/moe/moe_runner/marlin.py index df62ab6b78..5a61778f93 100644 --- a/python/sglang/srt/layers/moe/moe_runner/marlin.py +++ b/python/sglang/srt/layers/moe/moe_runner/marlin.py @@ -149,6 +149,11 @@ def fused_experts_none_to_marlin( f"Unsupported Marlin MoE activation: {runner_config.activation}" ) + # Marlin always applies the router weight inside its down GEMM. + assert ( + not runner_config.no_combine or runner_config.no_combine_keep_router_weight + ), "Marlin MoE supports no_combine only with no_combine_keep_router_weight" + # Use a per-call workspace so captured graphs cannot alias Marlin's # inter-block reduction locks and deadlock during capture. workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4) @@ -205,6 +210,7 @@ def fused_experts_none_to_marlin( gemm1_alpha=runner_config.gemm1_alpha, activation=runner_config.activation, is_gated=runner_config.is_gated, + no_combine=runner_config.no_combine, ).to(hidden_states.dtype) return StandardCombineInput( diff --git a/python/sglang/srt/layers/moe/moe_runner/triton.py b/python/sglang/srt/layers/moe/moe_runner/triton.py index 5939aaf181..30fd8c7c9b 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton.py @@ -162,6 +162,7 @@ class TritonRunnerCore(MoeRunnerCore): activation=self.config.activation, is_gated=self.config.is_gated, no_combine=self.config.no_combine, + no_combine_keep_router_weight=self.config.no_combine_keep_router_weight, inplace=self.config.inplace, apply_router_weight_on_input=self.config.apply_router_weight_on_input, routed_scaling_factor=self.config.routed_scaling_factor, diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index ad6b8c9d77..3e735b605a 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -262,6 +262,7 @@ def outplace_fused_experts( gate_up_interleaved: bool = True, a1_q: Optional[torch.Tensor] = None, fuse_swiglu_interleaved: bool = False, + no_combine_keep_router_weight: bool = False, ) -> torch.Tensor: return fused_experts_impl( hidden_states, @@ -296,6 +297,7 @@ def outplace_fused_experts( gate_up_interleaved=gate_up_interleaved, a1_q=a1_q, fuse_swiglu_interleaved=fuse_swiglu_interleaved, + no_combine_keep_router_weight=no_combine_keep_router_weight, ) @@ -395,6 +397,7 @@ def fused_experts( gate_up_interleaved=moe_runner_config.gate_up_interleaved, a1_q=a1_q, fuse_swiglu_interleaved=fuse_swiglu_interleaved, + no_combine_keep_router_weight=moe_runner_config.no_combine_keep_router_weight, ) @@ -556,6 +559,7 @@ def _fused_moe_kernel_sequence( gate_up_interleaved: bool = True, a1_q: Optional[torch.Tensor] = None, fuse_swiglu_interleaved: bool = False, + no_combine_keep_router_weight: bool = False, ) -> torch.Tensor: """Run the MoE kernel/activation/kernel/combine sequence in a single shot. @@ -911,7 +915,8 @@ def _fused_moe_kernel_sequence( sorted_token_ids, expert_ids, num_tokens_post_padded, - not apply_router_weight_on_input and not no_combine, + not apply_router_weight_on_input + and (not no_combine or no_combine_keep_router_weight), 1, down_config or config, compute_type=compute_type, @@ -935,6 +940,30 @@ def _fused_moe_kernel_sequence( del intermediate_cache2 + return moe_combine_topk( + intermediate_cache3=intermediate_cache3, + out_hidden_states=out_hidden_states, + routed_scaling_factor=routed_scaling_factor, + no_combine=no_combine, + use_fused_moe_sum_all_reduce=use_fused_moe_sum_all_reduce, + out_slice=out_slice, + _use_intermediate=_use_intermediate, + ) + + +def moe_combine_topk( + intermediate_cache3: torch.Tensor, + out_hidden_states: torch.Tensor, + routed_scaling_factor: Optional[float], + no_combine: bool = False, + use_fused_moe_sum_all_reduce: bool = False, + out_slice: Optional[torch.Tensor] = None, + _use_intermediate: bool = True, +) -> torch.Tensor: + """Sum the routed-weighted ``[num_tokens, topk, hidden]`` expert outputs into + ``out_hidden_states`` (the tail of ``_fused_moe_kernel_sequence``).""" + num_tokens, topk = intermediate_cache3.shape[:2] + if routed_scaling_factor is None: routed_scaling_factor = 1.0 @@ -1057,6 +1086,7 @@ def fused_experts_impl( gate_up_interleaved: bool = True, a1_q: Optional[torch.Tensor] = None, fuse_swiglu_interleaved: bool = False, + no_combine_keep_router_weight: bool = False, ): padded_size = padding_size if not (use_fp8_w8a8 or use_int8_w8a8) or block_shape is not None or _use_aiter: @@ -1137,6 +1167,7 @@ def fused_experts_impl( gate_up_interleaved=gate_up_interleaved, a1_q=a1_q, fuse_swiglu_interleaved=fuse_swiglu_interleaved, + no_combine_keep_router_weight=no_combine_keep_router_weight, ) diff --git a/python/sglang/srt/layers/moe/paged_experts/__init__.py b/python/sglang/srt/layers/moe/paged_experts/__init__.py new file mode 100644 index 0000000000..4f102fd914 --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/__init__.py @@ -0,0 +1,17 @@ +"""Paged experts: serve an MoE model whose routed experts do not fit in GPU memory. + +Each MoE layer keeps K of its E routed experts in a K-slot GPU table and pages the rest from +pinned host memory; outputs are bit-identical to the unpaged layer. See +``docs/docs/advanced_features/paged_experts.mdx`` for the design and the execution flow. + +Modules: ``formats`` (per quantization method), ``runners`` (per runner backend), ``store`` +(host copy and transfers), ``residency`` (slots and waves), ``executor`` (one forward step), +``method`` (the ``FusedMoEMethodBase`` wiring them into a ``FusedMoE`` layer). +""" + +from sglang.srt.layers.moe.paged_experts.method import ( + PagedExpertsMoEMethod, + make_for_layer, +) + +__all__ = ["PagedExpertsMoEMethod", "make_for_layer"] diff --git a/python/sglang/srt/layers/moe/paged_experts/executor.py b/python/sglang/srt/layers/moe/paged_experts/executor.py new file mode 100644 index 0000000000..473bf17412 --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/executor.py @@ -0,0 +1,137 @@ +"""Runs a paged forward step: page in, the wave's GEMMs, merge and finish per the runner +contract. + +``EagerExecutor`` decides residency on the host and serves any step, in as many waves as it +needs. ``DeviceExecutor`` decides and pages on the GPU without a host sync, so CUDA graphs can +capture it; it serves the steps whose routed entries fit the K slots (one wave), which covers +decode up to ``K // top_k`` requests. Both finish through the same contract. +""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from sglang.srt.layers.moe.paged_experts.residency import ( + DeviceResidency, + LRUPolicy, + plan_waves, +) +from sglang.srt.layers.moe.paged_experts.runners import RunnerContract, run_on_slots +from sglang.srt.layers.moe.paged_experts.store import HostExpertStore + + +class EagerExecutor: + def __init__( + self, + *, + base_method, + store: HostExpertStore, + policy: LRUPolicy, + contract: RunnerContract, + num_experts: int, + routed_scaling_factor: float, + device_residency: Optional[DeviceResidency] = None, + ): + self.base_method = base_method + self.store = store + self.policy = policy + self.contract = contract + self.num_experts = num_experts + self.routed_scaling_factor = routed_scaling_factor + # Shared with a DeviceExecutor, whose steps (and graph replays) move it on the GPU. + self.device_residency = device_residency + + def run(self, layer, dispatch_output): + if self.device_residency is not None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "Paged experts: a captured decode batch routes more entries than the K " + "resident slots; capture batch sizes must stay at or below K // top_k" + ) + self.device_residency.load_into(self.policy) + out = self._run(layer, dispatch_output) + if self.device_residency is not None: + self.device_residency.store_from(self.policy) + return out + + def _run(self, layer, dispatch_output): + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + topk_output = dispatch_output.topk_output + topk_ids = topk_output.topk_ids + distinct = [e for e in torch.unique(topk_ids).tolist() if e >= 0] + merged = None + waves = plan_waves(policy=self.policy, distinct=distinct) + for wave in waves: + self.store.page_in(layer=layer, loads=wave.loads) + # Index E (and -1, which wraps to it) stays -1: padded routing entries are masked. + to_slot = torch.full((self.num_experts + 1,), -1, dtype=torch.int32) + to_slot[wave.experts] = torch.tensor(wave.slots, dtype=torch.int32) + slots = to_slot.to(topk_ids.device)[topk_ids] + if len(waves) == 1: # only padding is outside the wave + merged = run_on_slots(self.base_method, layer, dispatch_output, slots) + break + merged = self.contract.run_wave( + base_method=self.base_method, + layer=layer, + dispatch_output=dispatch_output, + slots=slots, + merged=merged, + ) + return StandardCombineInput( + hidden_states=self.contract.finish( + merged=merged, + hidden_states=dispatch_output.hidden_states, + routed_scaling_factor=self.routed_scaling_factor, + ) + ) + + +class DeviceExecutor: + """One wave, decided and paged on the GPU: a step with at most K routed entries needs at + most K distinct experts, so they all fit the slots at once.""" + + def __init__( + self, + *, + base_method, + store: HostExpertStore, + residency: DeviceResidency, + contract: RunnerContract, + routed_scaling_factor: float, + ): + self.base_method = base_method + self.store = store + self.residency = residency + self.contract = contract + self.routed_scaling_factor = routed_scaling_factor + + def run(self, layer, dispatch_output): + from sglang.kernels.ops.moe.paged_experts import paged_experts_decide + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + r = self.residency + topk_ids = dispatch_output.topk_output.topk_ids + paged_experts_decide( + topk_ids.flatten().to(torch.int32), + r.step, + r.slot_expert, + r.expert_slot, + r.slot_lastuse, + r.src, + r.dst, + r.count, + ) + self.store.gather(src=r.src, dst=r.dst, count=r.count) + # Every routed expert is now resident; padding (-1) stays masked. + slots = r.expert_slot[topk_ids.clamp(min=0)].masked_fill(topk_ids < 0, -1) + partial = run_on_slots(self.base_method, layer, dispatch_output, slots) + return StandardCombineInput( + hidden_states=self.contract.finish( + merged=partial, + hidden_states=dispatch_output.hidden_states, + routed_scaling_factor=self.routed_scaling_factor, + ) + ) diff --git a/python/sglang/srt/layers/moe/paged_experts/formats.py b/python/sglang/srt/layers/moe/paged_experts/formats.py new file mode 100644 index 0000000000..5bed9c41d7 --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/formats.py @@ -0,0 +1,184 @@ +"""Per-quantization-method expert layout, one ``ExpertFormat`` per fused-MoE method. + +A format answers three questions for the method it wraps: + +* which per-expert tensors the checkpoint loads (``checkpoint_params``) and which are paged at + run time (``paged_params``); +* what happens after loading, so the host copy holds exactly what the GPU slots would hold + (for methods that repack their experts, the repack runs over the host copy K at a time); +* which runner contract (``runners.py``) its base method's runner follows. + +Adding a quantization method means adding a subclass to ``FORMATS``. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Optional, Tuple + +import torch + +from sglang.srt.layers.moe.paged_experts.runners import ( + MARLIN, + TRITON, + RunnerContract, +) +from sglang.srt.layers.moe.paged_experts.store import HostExpertStore + +# Prefixes of FusedMoE's per-expert parameters. +_EXPERT_PARAM_PREFIXES = ("w13_", "w2_") + + +class ExpertFormat(ABC): + #: Per-expert tensors the base method creates for loading. + checkpoint_params: Tuple[str, ...] = () + #: Per-expert tensors paged at run time. + paged_params: Tuple[str, ...] = () + #: Whether the base post-load step repacks the experts; it then runs over the host copy K + #: experts at a time. + repacks_after_loading: bool = False + contract: RunnerContract = TRITON + + @classmethod + @abstractmethod + def supports(cls, base_method) -> bool: + """Whether this format handles ``base_method``.""" + + def check_params(self, layer, num_slots: int) -> None: + """Raise unless the layer's per-expert tensors are exactly the declared ones.""" + per_expert = { + name + for name, p in layer.named_parameters(recurse=False) + if name.startswith(_EXPERT_PARAM_PREFIXES) + and p.dim() > 0 + and p.shape[0] == num_slots + } + if per_expert != set(self.checkpoint_params): + raise RuntimeError( + f"Paged experts: {type(self).__name__} expects per-expert parameters " + f"{sorted(self.checkpoint_params)}, the layer has {sorted(per_expert)}" + ) + + def after_loading(self, layer, base_method, store, num_slots: int): + """Run the base post-load step; return the store to page from, whose experts 0..K-1 + the GPU slots then hold.""" + if self.repacks_after_loading: + return self._repack( + layer=layer, + base_method=base_method, + staged=store, + num_slots=num_slots, + ) + for name, host in store.host.items(): + getattr(layer, name).data.copy_(host[:num_slots]) + base_method.process_weights_after_loading(layer) + # Paging copies host rows verbatim, so the base method must not transform the + # loaded weights (e.g. a kernel-specific shuffle or row interleave). + for name, host in store.host.items(): + gpu = getattr(layer, name).data[0] + if not torch.equal(gpu, host[0].to(gpu.device)): + raise RuntimeError( + f"Paged experts: {type(base_method).__name__} transforms {name} " + "after loading, which the host store does not replicate" + ) + return store + + def _repack(self, layer, base_method, staged, num_slots: int): + """Run the base post-load step over the staged checkpoint layout, K experts at a time, + collecting its per-expert output into a new store. Only valid for steps that treat + every expert independently.""" + num_experts = next(iter(staged.host.values())).shape[0] + templates = {name: getattr(layer, name) for name in staged.host} + packed = None + for start in range(0, num_experts, num_slots): + n = min(num_slots, num_experts - start) + # Fresh parameters in the checkpoint layout: the step may replace or rewrite them. + for name, template in templates.items(): + fresh = torch.nn.Parameter( + torch.empty_like(template.data), requires_grad=False + ) + fresh.__dict__.update(template.__dict__) + fresh.data[:n].copy_(staged.host[name][start : start + n]) + layer.register_parameter(name, fresh) + base_method.process_weights_after_loading(layer) + if packed is None: + packed = HostExpertStore( + layer=layer, names=self.paged_params, num_experts=num_experts + ) + for name, host in packed.host.items(): + host[start : start + n].copy_(getattr(layer, name).data[:n]) + for name, host in packed.host.items(): + getattr(layer, name).data.copy_(host[:num_slots]) + return packed + + +class UnquantizedFormat(ExpertFormat): + checkpoint_params = paged_params = ("w13_weight", "w2_weight") + + @classmethod + def supports(cls, base_method) -> bool: + from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod + + return isinstance(base_method, UnquantizedFusedMoEMethod) + + +class Fp8BlockFormat(ExpertFormat): + """FP8 weights with per-block scales (e.g. 128x128) and dynamic activation scales. On the + triton runner the post-load step leaves both untouched, so they page as stored.""" + + checkpoint_params = paged_params = ( + "w13_weight", + "w2_weight", + "w13_weight_scale_inv", + "w2_weight_scale_inv", + ) + + @classmethod + def supports(cls, base_method) -> bool: + from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod + + return ( + isinstance(base_method, Fp8MoEMethod) + and base_method.quant_config.weight_block_size is not None + and base_method.quant_config.is_checkpoint_fp8_serialized + and not base_method.use_mxfp8 + and not base_method.is_fp4_expert + ) + + +class GptqMarlinFormat(ExpertFormat): + """GPTQ int4/int8 experts on the Marlin runner. The post-load step sorts the act-order + indices and repacks each expert's weights and scales into Marlin's layout.""" + + checkpoint_params = ( + "w13_qweight", + "w2_qweight", + "w13_scales", + "w2_scales", + "w13_qzeros", + "w2_qzeros", + "w13_g_idx", + "w2_g_idx", + "w13_g_idx_sort_indices", + "w2_g_idx_sort_indices", + ) + # GPTQ-Marlin's MoE apply never passes the zero points to the kernel. + paged_params = tuple(n for n in checkpoint_params if not n.endswith("_qzeros")) + repacks_after_loading = True + contract = MARLIN + + @classmethod + def supports(cls, base_method) -> bool: + from sglang.srt.layers.quantization.gptq.gptq import GPTQMarlinMoEMethod + + return isinstance(base_method, GPTQMarlinMoEMethod) + + +FORMATS = (UnquantizedFormat, Fp8BlockFormat, GptqMarlinFormat) + + +def format_for(base_method) -> Optional[ExpertFormat]: + for fmt in FORMATS: + if fmt.supports(base_method): + return fmt() + return None diff --git a/python/sglang/srt/layers/moe/paged_experts/method.py b/python/sglang/srt/layers/moe/paged_experts/method.py new file mode 100644 index 0000000000..3ba9a5f542 --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/method.py @@ -0,0 +1,323 @@ +"""The fused-MoE method that wires paged experts into a ``FusedMoE`` layer.""" + +from __future__ import annotations + +import logging + +import torch + +from sglang.srt.layers.moe.paged_experts.executor import DeviceExecutor, EagerExecutor +from sglang.srt.layers.moe.paged_experts.formats import ExpertFormat, format_for +from sglang.srt.layers.moe.paged_experts.residency import DeviceResidency, LRUPolicy +from sglang.srt.layers.moe.paged_experts.store import HostExpertStore +from sglang.srt.layers.moe.topk import TopKOutputChecker +from sglang.srt.layers.quantization.base_config import FusedMoEMethodBase +from sglang.srt.model_executor.cuda_graph_config import ( + Backend, + Phase, + check_cuda_graph_backend, + cuda_graph_fully_disabled, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( + eager_on_graph, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( + is_in_breakable_cuda_graph, +) +from sglang.srt.runtime_context import ( + get_exec, + get_model, + get_parallel, + get_spec, + process_model_config, +) + +logger = logging.getLogger(__name__) + + +class PagedExpertsMoEMethod(FusedMoEMethodBase, torch.nn.Module): + """Wraps a fused-MoE method so it runs over a K-slot GPU table paged from host memory. + + An ``nn.Module`` like the methods it wraps, so it can replace one as a layer's child.""" + + def __init__( + self, + base_method, + expert_format: ExpertFormat, + num_experts: int, + num_resident: int, + graphs: bool = False, + ): + torch.nn.Module.__init__(self) + self.base_method = base_method + self.format = expert_format + self.num_experts = num_experts + self.num_resident = num_resident + self.policy = LRUPolicy(num_resident) + self.store = None # built once the base method has created the expert tensors + self.executor = None # built once the store holds the paged layout + # Some steps may run inside a CUDA graph: decide one-wave steps on the device, so + # the graph can capture them. + self.graphs = graphs + self.device_executor = None + self.routed_scaling_factor = 1.0 + + def create_weights( + self, + layer, + num_experts, + hidden_size, + intermediate_size_per_partition, + params_dtype, + **extra, + ): + # The GPU table has K rows. The loader below sends all E experts to the host store, + # so the native expert-id remap never sees ids >= K. + layer.num_local_experts = self.num_resident + layer._num_local_routed = layer._num_global_routed = self.num_resident + self.base_method.create_weights( + layer=layer, + num_experts=self.num_resident, + hidden_size=hidden_size, + intermediate_size_per_partition=intermediate_size_per_partition, + params_dtype=params_dtype, + **extra, + ) + # The loader follows the base method's weight layout, as it would without paging. + layer.weight_load_method = self.base_method + layer._derived_weight_cache_error = ( + "Online weight updates are not supported with --enable-paged-experts: " + "experts already on the GPU would keep the old weights." + ) + self.format.check_params(layer=layer, num_slots=self.num_resident) + # A store that only stages the checkpoint layout for a repack need not be pinned. + self.store = HostExpertStore( + layer=layer, + names=self.format.checkpoint_params, + num_experts=self.num_experts, + pin=not self.format.repacks_after_loading, + ) + for name in self.format.checkpoint_params: + getattr(layer, name).weight_loader = self._make_host_loader( + layer=layer, name=name + ) + + def _make_host_loader(self, layer, name): + def weight_loader(param, loaded_weight, weight_name, shard_id, expert_id): + # Reuse the native per-expert loader (sharding, w1/w3 placement, scale layout) + # on a one-row view of the host store, so host rows have exactly the native GPU + # layout. The row carries the parameter's loader tags (e.g. ``quant_method``). + row = torch.nn.Parameter( + self.store.host[name][expert_id : expert_id + 1], requires_grad=False + ) + row.__dict__.update(param.__dict__) + layer._weight_loader_impl( + param=row, + loaded_weight=loaded_weight, + weight_name=weight_name, + shard_id=shard_id, + expert_id=0, + ) + + return weight_loader + + def create_moe_runner(self, layer, moe_runner_config): + self.base_method.create_moe_runner( + layer=layer, + moe_runner_config=self.format.contract.runner_config( + moe_runner_config=moe_runner_config, num_slots=self.num_resident + ), + ) + self.runner = self.base_method.runner + self.format.contract.check(base_method=self.base_method) + self.routed_scaling_factor = moe_runner_config.routed_scaling_factor or 1.0 + + def process_weights_after_loading(self, layer): + self.store = self.format.after_loading( + layer=layer, + base_method=self.base_method, + store=self.store, + num_slots=self.num_resident, + ) + residency = None + if self.graphs: + residency = DeviceResidency( + num_experts=self.num_experts, + num_slots=self.num_resident, + device=getattr(layer, self.format.paged_params[0]).device, + ) + self.store.bind_device_gather(layer) + self.device_executor = DeviceExecutor( + base_method=self.base_method, + store=self.store, + residency=residency, + contract=self.format.contract, + routed_scaling_factor=self.routed_scaling_factor, + ) + self.executor = EagerExecutor( + base_method=self.base_method, + store=self.store, + policy=self.policy, + contract=self.format.contract, + num_experts=self.num_experts, + routed_scaling_factor=self.routed_scaling_factor, + device_residency=residency, + ) + + def apply(self, layer, dispatch_output): + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + # Decided from shapes alone, so a captured graph always takes the same path. + topk_output = dispatch_output.topk_output + if ( + self.device_executor is not None + and topk_output.topk_ids.numel() <= self.num_resident + ): + return self.device_executor.run( + layer=layer, dispatch_output=dispatch_output + ) + if is_in_breakable_cuda_graph(): + # The host-planned waves run as an eager break between graph segments. + assert ( + TopKOutputChecker.format_is_standard(topk_output) + and dispatch_output.hidden_states_scale is None + and dispatch_output.hidden_states_pre_quant is None + ), ( + "Paged experts: breakable CUDA graphs need standard, unquantized MoE inputs" + ) + out = torch.empty_like(dispatch_output.hidden_states) + self._run_eager_into( + layer, + dispatch_output.hidden_states, + topk_output.topk_weights, + topk_output.topk_ids, + topk_output.router_logits, + out, + ) + return StandardCombineInput(hidden_states=out) + return self.executor.run(layer=layer, dispatch_output=dispatch_output) + + def _run_eager_into_impl( + self, layer, hidden_states, topk_weights, topk_ids, router_logits, out + ) -> None: + # Flat tensors in, output buffer filled in place: a break replays with the tensors it + # was captured with, rebuilt into the dispatch output here. + from sglang.srt.layers.moe.token_dispatcher import StandardDispatchOutput + from sglang.srt.layers.moe.topk import StandardTopKOutput + + dispatch_output = StandardDispatchOutput( + hidden_states=hidden_states, + hidden_states_scale=None, + topk_output=StandardTopKOutput( + topk_weights=topk_weights, + topk_ids=topk_ids, + router_logits=router_logits, + ), + ) + out.copy_( + self.executor.run( + layer=layer, dispatch_output=dispatch_output + ).hidden_states + ) + + _run_eager_into = eager_on_graph(True)(_run_eager_into_impl) + + +def make_for_layer(layer, base_method) -> PagedExpertsMoEMethod: + expert_format = format_for(base_method) + check_paged_experts_compat(layer=layer, expert_format=expert_format) + num_experts = layer.num_local_experts + num_resident = min(get_exec().moe.paged_experts_num_resident, num_experts) + if check_cuda_graph_backend(Phase.DECODE, Backend.FULL): + # A full decode graph cannot break out for host-planned waves. + _cap_decode_capture_batch_sizes(max(1, num_resident // _router_top_k(layer))) + return PagedExpertsMoEMethod( + base_method=base_method, + expert_format=expert_format, + num_experts=num_experts, + num_resident=num_resident, + graphs=not cuda_graph_fully_disabled(), + ) + + +def _cap_decode_capture_batch_sizes(max_bs: int) -> None: + """Capture full decode graphs only at batch sizes whose routed entries fit the K slots, + the steps the device executor serves; larger batches run without a graph. The cap itself + is captured, so batches up to it pad to a graph. Every MoE layer calls this; only the + first changes the config.""" + decode = get_exec().graph.cuda_graph_config.decode + if not decode.bs: + return + cap = min(max_bs, max(decode.bs)) + capture_bs = sorted({bs for bs in decode.bs if bs <= cap} | {cap}) + if capture_bs != list(decode.bs): + logger.info( + "Paged experts: decode CUDA graphs captured at batch sizes %s (at most K // top_k)", + capture_bs, + ) + decode.bs = capture_bs + decode.max_bs = cap + + +def check_paged_experts_compat(layer, expert_format) -> None: + """Raise if the server or layer configuration is outside what paged experts supports; + ``expert_format`` is None when no format handles the layer's quantization method.""" + + problems = [] + parallel = get_parallel() + for name in ("tp_size", "ep_size", "pp_size", "dp_size"): + if getattr(parallel, name) > 1: + problems.append(f"--{name.replace('_', '-')} must be 1 (single GPU only)") + if get_exec().moe.enable_eplb: + problems.append("--enable-eplb is not supported") + if get_spec().speculative_algorithm: + problems.append("speculative decoding is not supported") + # Host-planned waves run as eager breaks in a breakable prefill graph; a full graph only + # captures steps that fit one wave, which decode can be limited to and prefill cannot. + if not ( + check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE) + or check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED) + ): + problems.append( + "prefill CUDA graphs must use the breakable backend or be disabled" + ) + if not ( + check_cuda_graph_backend(Phase.DECODE, Backend.FULL) + or check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED) + ): + problems.append("decode CUDA graphs must use the full backend or be disabled") + if get_model().load_format == "dummy": + problems.append( + "--load-format dummy is not supported: the host store needs real weights" + ) + num_resident = get_exec().moe.paged_experts_num_resident + top_k = _router_top_k(layer) + if num_resident is None: + problems.append("--paged-experts-num-resident is required") + elif num_resident < top_k: + problems.append( + f"--paged-experts-num-resident must be at least the router's top-k ({top_k})" + ) + if expert_format is None: + problems.append( + "only unquantized (bf16/fp16), block-quantized FP8 and GPTQ experts are " + f"supported, got {type(layer.quant_method).__name__}" + ) + if get_exec().moe.enable_fused_moe_sum_all_reduce: + problems.append( + "--enable-fused-moe-sum-all-reduce is not supported: paged experts sums the " + "top-k outputs itself" + ) + if layer.num_fused_shared_experts: + problems.append( + "fused shared experts are not supported (--disable-shared-experts-fusion)" + ) + if problems: + raise RuntimeError("Paged experts: " + "; ".join(problems)) + + +def _router_top_k(layer) -> int: + # Models that route through a separate TopK module (e.g. OLMoE) leave FusedMoE's unset. + if layer.top_k is not None: + return layer.top_k + return process_model_config().hf_text_config.num_experts_per_tok diff --git a/python/sglang/srt/layers/moe/paged_experts/residency.py b/python/sglang/srt/layers/moe/paged_experts/residency.py new file mode 100644 index 0000000000..41d40a5dcc --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/residency.py @@ -0,0 +1,99 @@ +"""Which expert sits in which GPU slot, and how a step's experts are split into waves. + +Pure bookkeeping: no expert moves here. The executor pages ``Wave.loads`` into place before it +runs the wave. With CUDA graphs, decode steps decide on the device (``DeviceResidency``), and +host-planned steps load that state into the policy and store it back. +""" + +from __future__ import annotations + +from collections import OrderedDict +from typing import List, Tuple + +import msgspec +import torch + + +class Wave(msgspec.Struct, frozen=True): + experts: List[int] # logical expert ids served by this wave + slots: List[int] # the GPU slot of each expert, same order + loads: List[Tuple[int, int]] # (expert, slot) pairs to page in before the wave runs + + +class LRUPolicy: + """Chooses the slot of each expert of a wave, evicting the least recently used experts + outside it. Slots start holding experts 0..K-1.""" + + def __init__(self, num_slots: int): + self.num_slots = num_slots + # Logical expert -> slot for the resident experts, least recently used first. + self.resident = OrderedDict((e, e) for e in range(num_slots)) + + def place(self, experts: List[int]) -> Wave: + """Make ``experts`` (at most ``num_slots``) resident and return where they are.""" + for e in experts: + if e in self.resident: + self.resident.move_to_end(e) + loads = [] + for e in experts: + if e not in self.resident: + # Every expert of this wave sits at the LRU tail, so the head is never one of them. + _, slot = self.resident.popitem(last=False) + self.resident[e] = slot + loads.append((e, slot)) + return Wave( + experts=experts, slots=[self.resident[e] for e in experts], loads=loads + ) + + +def plan_waves(policy: LRUPolicy, distinct: List[int]) -> List[Wave]: + """Split a step's distinct experts into waves of at most ``num_slots`` and place each. + + A step with no routed expert (all padding) still gets one empty wave, so the layer runs. + """ + k = policy.num_slots + ordered = [e for e in distinct if e in policy.resident] + [ + e for e in distinct if e not in policy.resident + ] + groups = [ordered[i : i + k] for i in range(0, len(ordered), k)] or [[]] + return [policy.place(group) for group in groups] + + +class DeviceResidency: + """The LRU residency state on the GPU, which the decide kernel updates in place, plus the + buffers of its page-in plan. Slots start holding experts 0..K-1, like ``LRUPolicy``.""" + + def __init__(self, num_experts: int, num_slots: int, device): + i32 = dict(dtype=torch.int32, device=device) + self.step = torch.zeros(1, **i32) + self.slot_expert = torch.arange(num_slots, **i32) + self.expert_slot = torch.full((num_experts,), -1, **i32) + self.expert_slot[:num_slots] = self.slot_expert + self.slot_lastuse = torch.zeros(num_slots, **i32) + self.src = torch.zeros(num_slots, **i32) + self.dst = torch.zeros(num_slots, **i32) + self.count = torch.zeros(1, **i32) + + def load_into(self, policy: LRUPolicy) -> None: + """Make ``policy`` hold this state: the same residents, least recently used first.""" + experts = self.slot_expert.tolist() + lastuse = self.slot_lastuse.tolist() + order = sorted(range(len(experts)), key=lambda s: (lastuse[s], s)) + policy.resident = OrderedDict((experts[s], s) for s in order) + + def store_from(self, policy: LRUPolicy) -> None: + """Make this state hold ``policy``'s, its recency order as the last steps.""" + num_slots = self.slot_expert.numel() + base = int(self.step.item()) + slot_expert = [0] * num_slots + lastuse = [0] * num_slots + for rank, (expert, slot) in enumerate(policy.resident.items(), start=1): + slot_expert[slot] = expert + lastuse[slot] = base + rank + self.slot_expert.copy_(torch.tensor(slot_expert, dtype=torch.int32)) + self.slot_lastuse.copy_(torch.tensor(lastuse, dtype=torch.int32)) + self.expert_slot.fill_(-1) + self.expert_slot[self.slot_expert.long()] = torch.arange( + num_slots, dtype=torch.int32, device=self.expert_slot.device + ) + self.step.fill_(base + num_slots) diff --git a/python/sglang/srt/layers/moe/paged_experts/runners.py b/python/sglang/srt/layers/moe/paged_experts/runners.py new file mode 100644 index 0000000000..f80ba6fb0a --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/runners.py @@ -0,0 +1,146 @@ +"""What paged experts needs from a fused-MoE runner backend, one ``RunnerContract`` per backend. + +The executor runs a step in waves (one wave when its experts fit the K slots). The runner +returns every routed (token, expert) output already multiplied by its router weight +(``no_combine``); the contract runs each wave over only the entries whose expert it holds, +writes them into one merged buffer, and performs the top-k sum the runner would have performed +itself. A quantization format names the contract of the runner its base method uses; supporting +a new backend means adding a contract here. +""" + +from __future__ import annotations + +from dataclasses import replace + +import torch +import torch.nn.functional as F + + +def run_on_slots(base_method, layer, dispatch_output, slots, outside: int = 0): + """Run the base method over the K-slot table: routed entry i goes to GPU slot ``slots[i]``; + entries at -1 get expert id ``outside`` and router weight 0, so they contribute nothing.""" + masked = slots < 0 + topk_output = dispatch_output.topk_output + wave_output = dispatch_output._replace( + topk_output=topk_output._replace( + topk_ids=slots.masked_fill(masked, outside), + topk_weights=topk_output.topk_weights.masked_fill(masked, 0), + ), + ) + return base_method.apply(layer=layer, dispatch_output=wave_output).hidden_states + + +class RunnerContract: + def runner_config(self, moe_runner_config, num_slots: int): + """The base runner's config: K local experts, router-weighted per-expert outputs.""" + return replace( + moe_runner_config, + num_local_experts=num_slots, + inplace=False, + no_combine=True, + no_combine_keep_router_weight=True, + ) + + def check(self, base_method) -> None: + """Raise if the base method's runner cannot honor ``runner_config``.""" + + def run_wave(self, base_method, layer, dispatch_output, slots, merged): + """Compute the routed entries of one wave (``slots`` >= 0) into ``merged``, the step's + ``[tokens, top_k, hidden]`` entries (None before its first wave), and return it.""" + raise NotImplementedError + + def finish(self, merged, hidden_states, routed_scaling_factor: float): + """The layer output: the top-k sum of the merged entries.""" + out = torch.empty_like(hidden_states) + self._sum_top_k(merged, out, routed_scaling_factor) + return out + + def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None: + """Sum ``[tokens, top_k, hidden]`` into ``out`` exactly as the runner would.""" + raise NotImplementedError + + +class TritonContract(RunnerContract): + def check(self, base_method) -> None: + backend = base_method.runner.runner_backend + if not backend.is_triton(): + raise RuntimeError( + f"Paged experts: requires the triton MoE runner, got {backend.value}" + ) + + def run_wave(self, base_method, layer, dispatch_output, slots, merged): + config = base_method.runner.config + skips = ( + config.num_experts is None or config.num_experts != config.num_local_experts + ) + partial = run_on_slots( + base_method, layer, dispatch_output, slots, outside=-1 if skips else 0 + ) + if merged is None: + return partial + outside = (slots < 0).unsqueeze(-1) + return torch.where(outside, merged, partial, out=merged) + + def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None: + from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( + moe_combine_topk, + ) + + moe_combine_topk( + intermediate_cache3=entries, + out_hidden_states=out, + routed_scaling_factor=routed_scaling_factor, + ) + + +class MarlinContract(RunnerContract): + # The Marlin MoE methods build a Marlin runner unconditionally, so there is nothing to check. + + def run_wave(self, base_method, layer, dispatch_output, slots, merged): + # Only the wave's entries, compacted: the runner sees no masked rows. + hidden_states = dispatch_output.hidden_states + assert ( + dispatch_output.hidden_states_scale is None + and dispatch_output.hidden_states_pre_quant is None + ), "Paged experts: compacted waves need unquantized MoE inputs" + if merged is None: + merged = hidden_states.new_zeros(*slots.shape, hidden_states.shape[-1]) + entries = (slots >= 0).flatten().nonzero().squeeze(1) + num = entries.numel() + if num == 0: + return merged + # Padded to a few sizes per power of two, so the runner sees few distinct batch + # shapes; padding rows go to slot 0 with weight 0 and are dropped. + step = max(16, 1 << max(0, num.bit_length() - 4)) + pad = -num % step + tokens = F.pad(entries // slots.shape[1], (0, pad)) + topk = dispatch_output.topk_output + compact = dispatch_output._replace( + hidden_states=hidden_states[tokens], + topk_output=topk._replace( + topk_ids=F.pad(slots.flatten()[entries], (0, pad)).unsqueeze(1), + topk_weights=F.pad( + topk.topk_weights.flatten()[entries], (0, pad) + ).unsqueeze(1), + router_logits=( + None if topk.router_logits is None else topk.router_logits[tokens] + ), + ), + ) + out = base_method.apply(layer=layer, dispatch_output=compact).hidden_states + merged.view(-1, merged.shape[-1])[entries] = out.view(-1, merged.shape[-1])[ + :num + ] + return merged + + def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None: + # The reduce fused_marlin_moe itself ends with. + from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import ( + moe_sum_reduce, + ) + + moe_sum_reduce(entries, out, routed_scaling_factor) + + +TRITON = TritonContract() +MARLIN = MarlinContract() diff --git a/python/sglang/srt/layers/moe/paged_experts/store.py b/python/sglang/srt/layers/moe/paged_experts/store.py new file mode 100644 index 0000000000..301e4c7930 --- /dev/null +++ b/python/sglang/srt/layers/moe/paged_experts/store.py @@ -0,0 +1,71 @@ +"""The host copy of every expert and the transfer of experts into GPU slots.""" + +from __future__ import annotations + +from typing import Dict, Sequence, Tuple + +import torch + + +class HostExpertStore: + """Holds all E experts of each paged tensor on the host, in the layer's GPU layout. + Page-locked (``pin``) when it serves paging; a store only staged for a post-load repack + stays pageable. ``page_in`` is planned on the host; ``gather`` reads a plan on the GPU. + + The GPU tensors are looked up on the layer at every transfer, because a quantization + method may rebind them after loading. + """ + + def __init__(self, layer, names: Sequence[str], num_experts: int, pin: bool = True): + self.pin = pin + self.host: Dict[str, torch.Tensor] = { + name: self._alloc( + shape=(num_experts, *getattr(layer, name).shape[1:]), + dtype=getattr(layer, name).dtype, + ) + for name in names + } + + def _alloc(self, shape, dtype: torch.dtype) -> torch.Tensor: + return torch.empty(shape, dtype=dtype, device="cpu", pin_memory=self.pin) + + def page_in(self, layer, loads: Sequence[Tuple[int, int]]) -> None: + """Copy each ``(expert, slot)`` pair's expert, of every paged tensor, into its slot.""" + for name, host in self.host.items(): + gpu = getattr(layer, name).data + for expert, slot in loads: + gpu[slot].copy_(host[expert], non_blocking=True) + + def bind_device_gather(self, layer) -> None: + """Prepare ``gather``; called once the layer's tensors are final. The gather kernel + reads the pinned store directly, through its UVA address.""" + from sglang.kernels.ops.moe.paged_experts import ( + paged_experts_host_device_pointer, + ) + + hosts, gpus, words = [], [], [] + for name, host in self.host.items(): + gpu = getattr(layer, name).data + row_bytes = host[0].numel() * host.element_size() + if row_bytes == 0: + continue # e.g. GPTQ without act-order keeps empty g_idx tensors + if not (self.pin and gpu.is_contiguous() and row_bytes % 4 == 0): + raise RuntimeError( + f"Paged experts: CUDA graphs need pinned, contiguous expert rows of whole " + f"4-byte words; {name} has {row_bytes} bytes per expert, " + f"pinned={host.is_pinned()}, contiguous={gpu.is_contiguous()}" + ) + hosts.append(paged_experts_host_device_pointer(host)) + gpus.append(gpu.data_ptr()) + words.append(row_bytes // 4) + self._gather_args = [ + torch.tensor(v, dtype=torch.int64, device=gpu.device) + for v in (hosts, gpus, words) + ] + + def gather(self, src: torch.Tensor, dst: torch.Tensor, count: torch.Tensor) -> None: + """Copy expert ``src[i]`` into GPU slot ``dst[i]`` for ``i < count[0]``, with every + index read on the GPU (a device-planned step): capturable in a CUDA graph.""" + from sglang.kernels.ops.moe.paged_experts import paged_experts_gather + + paged_experts_gather(*self._gather_args, src, dst, count) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 72ceef49c1..9ae9d18c21 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -850,6 +850,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module): self.alt_stream is not None and get_is_capture_mode() and not torch.compiler.is_compiling() + # Paged experts run as an eager break, which must end the segment on the + # capture stream. + and not ( + get_exec().moe.enable_paged_experts and is_in_breakable_cuda_graph() + ) ): final_hidden_states, shared_output = self.forward_normal_dual_stream( hidden_states, diff --git a/test/registered/e2e/moe/test_paged_experts.py b/test/registered/e2e/moe/test_paged_experts.py new file mode 100644 index 0000000000..1f3b415dff --- /dev/null +++ b/test/registered/e2e/moe/test_paged_experts.py @@ -0,0 +1,166 @@ +"""Paged experts match the unpaged model, with decode captured (prompts one at a time) and host-planned (one batch).""" + +import statistics +import unittest +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE, + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + find_available_port, + popen_launch_server, + terminate_and_kill_process_tree, +) + +register_cuda_ci(est_time=360, stage="base-b", runner_config="1-gpu-large") + +PROMPTS = [ + "Natalia sold clips to 48 of her friends in April, and then she sold half as many " + "clips in May. How many clips did Natalia sell altogether in April and May?", + "def fibonacci(n):\n if n < 2:\n return n\n", + "The ocean's waves crash against the shore in an endless rhythm,", +] +# Decode graphs only at bs 1, so a batch of prompts decodes without one; prefill graphs only at +# the sizes the prompts need. +SERVER_ARGS = [ + "--cuda-graph-backend-decode", + "full", + "--cuda-graph-backend-prefill", + "breakable", + "--cuda-graph-max-bs-decode", + "1", + "--cuda-graph-bs-prefill", + "64", + "128", + "--skip-server-warmup", + # Small enough that the unpaged and paged servers share one GPU. + "--max-total-tokens", + "4096", + "--mem-fraction-static", + "0.6", + "--disable-shared-experts-fusion", + "--disable-radix-cache", + "--enable-deterministic-inference", + "--attention-backend", + "triton", +] +PAGED_ARGS = ["--enable-paged-experts", "--paged-experts-num-resident", "8"] +_HOST, _PORT = DEFAULT_URL_FOR_TEST.rsplit(":", 1) +PAGED_URL = f"{_HOST}:{find_available_port(int(_PORT) + 100)}" + + +class _PagedMatchesUnpaged: + model: str + #: False for MoE kernels that are not batch-invariant: compare the median logprob difference + #: per prompt, decode ones up to the first greedy token the servers differ on. + exact = True + median_abs_tolerance = 0.0 + + @classmethod + def setUpClass(cls): + # Both servers boot and score at the same time. + with ThreadPoolExecutor(2) as pool: + reference = pool.submit( + _serve_and_score, cls.model, DEFAULT_URL_FOR_TEST, [] + ) + paged = pool.submit(_serve_and_score, cls.model, PAGED_URL, PAGED_ARGS) + cls.reference, cls.paged = reference.result(), paged.result() + + def test_captured_decode_matches(self): + self._assert_match(self.reference["single"], self.paged["single"]) + + def test_host_planned_decode_matches(self): + self._assert_match(self.reference["batch"], self.paged["batch"]) + + def _assert_match(self, reference, paged): + decode_compared = decode_total = 0 + for prompt, ref, got in zip(PROMPTS, reference, paged): + with self.subTest(prompt=prompt[:30]): + if self.exact: + self.assertEqual(got, ref) + continue + diffs = [abs(a - b) for a, b in zip(got[0], ref[0]) if a is not None] + for (ref_tok, ref_lp), (tok, lp) in zip(ref[1], got[1]): + if tok != ref_tok: + break + diffs.append(abs(lp - ref_lp)) + decode_compared += 1 + decode_total += len(ref[1]) + self.assertLessEqual( + statistics.median(diffs), self.median_abs_tolerance + ) + if not self.exact: + self.assertGreaterEqual(decode_compared, decode_total // 2) + + +class TestPagedExpertsBf16(_PagedMatchesUnpaged, CustomTestCase): + model = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE + + +class TestPagedExpertsFp8Block(_PagedMatchesUnpaged, CustomTestCase): + """Block-FP8 experts (128x128 scales) on the triton runner.""" + + model = DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE + + +class TestPagedExpertsGptq(_PagedMatchesUnpaged, CustomTestCase): + """GPTQ int4 experts on the Marlin runner, whose reduction depends on the batch shape: the + unpaged server alone differs by up to 4 nats at a near-tied position between one prompt and + a batch, with medians below 0.05.""" + + model = "Qwen/Qwen3-30B-A3B-GPTQ-Int4" + exact = False + median_abs_tolerance = 0.1 + + +def _serve_and_score(model, base_url, other_args): + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=SERVER_ARGS + other_args, + # Compile DeepGEMM kernels lazily for the few shapes the prompts use. + env={"SGLANG_JIT_DEEPGEMM_PRECOMPILE": "0"}, + ) + try: + return { + "single": [_generate(base_url, [p])[0] for p in PROMPTS], + "batch": _generate(base_url, PROMPTS), + } + finally: + terminate_and_kill_process_tree(process) + + +def _generate(base_url, prompts): + """Prompt logprobs and greedy decode (token, logprob) pairs for ``prompts``, sent as one + batched request.""" + metas = requests.post( + base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16, + "ignore_eos": True, + }, + "return_logprob": True, + "logprob_start_len": 0, + }, + ).json() + return [ + ( + [lp for lp, _, _ in m["meta_info"]["input_token_logprobs"]], + [(tok, lp) for lp, tok, _ in m["meta_info"]["output_token_logprobs"]], + ) + for m in metas + ] + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/test/registered/kernels/ops/moe/test_paged_experts_kernels.py b/test/registered/kernels/ops/moe/test_paged_experts_kernels.py new file mode 100644 index 0000000000..6aca2c27c8 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_paged_experts_kernels.py @@ -0,0 +1,170 @@ +import unittest + +import torch + +from sglang.kernels.ops.moe.paged_experts import ( + paged_experts_decide, + paged_experts_gather, + paged_experts_host_device_pointer, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +E, K = 64, 16 + + +class _State: + """Residency state on the GPU, slots starting with experts 0..K-1.""" + + def __init__(self): + i32 = dict(dtype=torch.int32, device="cuda") + self.step = torch.zeros(1, **i32) + self.slot_expert = torch.arange(K, **i32) + self.expert_slot = torch.full((E,), -1, **i32) + self.expert_slot[:K] = self.slot_expert + self.slot_lastuse = torch.zeros(K, **i32) + self.src = torch.zeros(K, **i32) + self.dst = torch.zeros(K, **i32) + self.count = torch.zeros(1, **i32) + + def decide(self, topk): + paged_experts_decide( + topk, + self.step, + self.slot_expert, + self.expert_slot, + self.slot_lastuse, + self.src, + self.dst, + self.count, + ) + + +class _Reference: + """The decision the kernel must make, in Python.""" + + def __init__(self): + self.step = 0 + self.slot_expert = list(range(K)) + self.slot_lastuse = [0] * K + + def decide(self, topk): + self.step += 1 + for e in topk: + if e in self.slot_expert: + self.slot_lastuse[self.slot_expert.index(e)] = self.step + plan = [] + for e in topk: + if e < 0 or e in self.slot_expert: + continue + free = [s for s in range(K) if self.slot_lastuse[s] != self.step] + victim = min(free, key=lambda s: (self.slot_lastuse[s], s)) + self.slot_expert[victim] = e + self.slot_lastuse[victim] = self.step + plan.append((e, victim)) + return plan + + +def _random_topk(generator, num_entries): + topk = torch.randint(0, E, (num_entries,), generator=generator, dtype=torch.int32) + topk[torch.rand(num_entries, generator=generator) < 0.1] = -1 # padding + return topk + + +@unittest.skipUnless(torch.cuda.is_available(), "paged experts kernels need CUDA") +class TestPagedExpertsKernels(CustomTestCase): + def assert_state(self, state, ref, plan): + n = int(state.count.item()) + got_plan = list(zip(state.src[:n].tolist(), state.dst[:n].tolist())) + self.assertEqual(got_plan, plan) + self.assertEqual(state.slot_expert.tolist(), ref.slot_expert) + self.assertEqual(state.slot_lastuse.tolist(), ref.slot_lastuse) + expert_slot = [-1] * E + for s, e in enumerate(ref.slot_expert): + expert_slot[e] = s + self.assertEqual(state.expert_slot.tolist(), expert_slot) + + def test_decide_matches_reference(self): + generator = torch.Generator().manual_seed(0) + state, ref = _State(), _Reference() + for _ in range(200): + topk = _random_topk(generator, int(torch.randint(1, K + 1, (1,)))) + plan = ref.decide(topk.tolist()) + state.decide(topk.cuda()) + self.assert_state(state, ref, plan) + + def test_decide_rejects_more_entries_than_slots(self): + with self.assertRaises(Exception): + _State().decide(torch.zeros(K + 1, dtype=torch.int32, device="cuda")) + + def test_gather_copies_the_plan_for_every_tensor(self): + hosts = [ + torch.randn(E, 8, 32, dtype=torch.bfloat16).pin_memory(), + torch.randint(0, 100, (E, 12), dtype=torch.int32).pin_memory(), + torch.randn(E, dtype=torch.float32).pin_memory(), # 4-byte rows + torch.randn(E, 3, dtype=torch.float32).pin_memory(), # 12-byte rows + ] + gpus = [ + torch.zeros(K, *h.shape[1:], dtype=h.dtype, device="cuda") for h in hosts + ] + args = _gather_args(hosts, gpus) + src = torch.tensor([5, 63, 17] + [0] * (K - 3), dtype=torch.int32).cuda() + dst = torch.tensor([2, 0, 15] + [0] * (K - 3), dtype=torch.int32).cuda() + count = torch.tensor([3], dtype=torch.int32, device="cuda") + + paged_experts_gather(*args, src, dst, count) + + for host, gpu in zip(hosts, gpus): + for e, s in ((5, 2), (63, 0), (17, 15)): + self.assertTrue(torch.equal(gpu[s].cpu(), host[e])) + self.assertFalse(gpu[1].any()) # untouched + + def test_graph_replays_decide_on_new_routing(self): + """A captured decide + gather plans and pages each replay's routing, not the routing + seen at capture.""" + host = torch.randn(E, 64, dtype=torch.float32).pin_memory() + gpu = torch.zeros(K, 64, device="cuda") + gpu.copy_(host[:K]) + args = _gather_args([host], [gpu]) + state, ref = _State(), _Reference() + topk = torch.zeros(K // 2, dtype=torch.int32, device="cuda") + + def step(): + state.decide(topk) + paged_experts_gather(*args, state.src, state.dst, state.count) + + topk.copy_(torch.arange(K // 2, dtype=torch.int32)) + ref.decide(topk.tolist()) + step() # warm up outside the capture + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + step() + ref.decide(topk.tolist()) # capture does not run the kernels, ... + graph.replay() # ... so this replay is the second decide of the same routing + + generator = torch.Generator().manual_seed(1) + for _ in range(20): + routing = _random_topk(generator, K // 2) + topk.copy_(routing) + plan = ref.decide(routing.tolist()) + graph.replay() + self.assert_state(state, ref, plan) + for s, e in enumerate(ref.slot_expert): + self.assertTrue(torch.equal(gpu[s].cpu(), host[e])) + + +def _gather_args(hosts, gpus): + return [ + torch.tensor(v, dtype=torch.int64, device="cuda") + for v in ( + [paged_experts_host_device_pointer(h) for h in hosts], + [g.data_ptr() for g in gpus], + [h[0].numel() * h.element_size() // 4 for h in hosts], + ) + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/paged_experts/test_paged_experts.py b/test/registered/unit/layers/moe/paged_experts/test_paged_experts.py new file mode 100644 index 0000000000..570695aac5 --- /dev/null +++ b/test/registered/unit/layers/moe/paged_experts/test_paged_experts.py @@ -0,0 +1,369 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe.paged_experts import PagedExpertsMoEMethod +from sglang.srt.layers.moe.paged_experts.formats import ExpertFormat, UnquantizedFormat +from sglang.srt.layers.moe.paged_experts.method import ( + _cap_decode_capture_batch_sizes, + check_paged_experts_compat, +) +from sglang.srt.layers.moe.paged_experts.residency import ( + DeviceResidency, + LRUPolicy, + plan_waves, +) +from sglang.srt.layers.moe.paged_experts.runners import MarlinContract, TritonContract +from sglang.srt.layers.moe.paged_experts.store import HostExpertStore +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +E, K, H, TOP_K = 16, 4, 8, 2 + + +class _LinearMoE: + """Stand-in for the fused-MoE method in no-combine mode: the [tokens, top_k, hidden] + entries w[t,j] * x[t] @ w13[id] @ w2[id], left for the caller to sum; expert id -1 gives + zero entries.""" + + runner = SimpleNamespace(config=SimpleNamespace(num_experts=E, num_local_experts=K)) + + def __init__(self): + self.rows = 0 + + def apply(self, layer, dispatch_output): + topk = dispatch_output.topk_output + ids, w = topk.topk_ids.long(), topk.topk_weights + valid = ids >= 0 + self.rows += int(valid.sum()) + ids = ids.clamp(min=0) + y = torch.einsum( + "th,tjhi->tji", dispatch_output.hidden_states, layer.w13_weight[ids] + ) + y = torch.einsum("tji,tjio->tjo", y, layer.w2_weight[ids]) + return SimpleNamespace(hidden_states=(w * valid).unsqueeze(-1) * y) + + def process_weights_after_loading(self, layer): + pass + + +class _CpuStore(HostExpertStore): + def _alloc(self, shape, dtype): + return torch.empty(shape, dtype=dtype) + + +def _sum_top_k(self, entries, out, routed_scaling_factor): + out.copy_(entries.sum(1) * routed_scaling_factor) + + +class _CompactingContract(MarlinContract): + _sum_top_k = _sum_top_k + + +class _SkippingContract(TritonContract): + _sum_top_k = _sum_top_k + + +CONTRACTS = (_CompactingContract(), _SkippingContract()) + + +def _reference(full, dispatch_output): + return _LinearMoE().apply(full, dispatch_output).hidden_states.sum(1) + + +def _step(num_tokens, padded=False): + """A random routing step; ``padded`` marks the first token's entries as padding (-1).""" + ids = torch.stack([torch.randperm(E)[:TOP_K] for _ in range(num_tokens)]).int() + if padded: + ids[0] = -1 + weights = torch.rand(num_tokens, TOP_K, dtype=torch.float32) + x = torch.randn(num_tokens, H, dtype=torch.float32) + return StandardDispatchOutput( + hidden_states=x, + hidden_states_scale=None, + topk_output=StandardTopKOutput( + topk_weights=weights, topk_ids=ids, router_logits=None + ), + ) + + +def _paged_layer(contract=CONTRACTS[0], extra_params=(), load=True): + """A paged method over random experts, its layer created through the base method on a CPU + store; ``load`` fills the store with the experts and runs the post-load step.""" + torch.manual_seed(0) + full = {name: torch.randn(E, H, H) for name in ("w13_weight", "w2_weight")} + + class _Base(_LinearMoE): + def create_weights(self, layer, num_experts, **kwargs): + for name in ("w13_weight", "w2_weight", *extra_params): + param = torch.nn.Parameter( + torch.zeros(num_experts, H, H), requires_grad=False + ) + param.quant_method = "block" + layer.register_parameter(name, param) + + expert_format = UnquantizedFormat() + expert_format.contract = contract + method = PagedExpertsMoEMethod( + base_method=_Base(), expert_format=expert_format, num_experts=E, num_resident=K + ) + layer = torch.nn.Module() + layer.weight_load_method = None + with patch("sglang.srt.layers.moe.paged_experts.method.HostExpertStore", _CpuStore): + method.create_weights( + layer=layer, + num_experts=E, + hidden_size=H, + intermediate_size_per_partition=H, + params_dtype=torch.float32, + ) + if load: + for name, t in full.items(): + method.store.host[name].copy_(t) + method.process_weights_after_loading(layer) + return method, layer, SimpleNamespace(**full) + + +class TestPagedForward(CustomTestCase): + def test_matches_unpaged(self): + """Consecutive steps (which reuse and evict residents), in one wave (1, 2 tokens) or + several (64 tokens), with and without padded entries: each output equals the unpaged + layer, the slots hold the resident experts, and no wave computes the entries of + another (without compaction, every wave would compute all of them).""" + for contract in CONTRACTS: + for num_tokens, padded in ((1, False), (2, False), (64, False), (3, True)): + with self.subTest( + contract=type(contract).__name__, + num_tokens=num_tokens, + padded=padded, + ): + method, layer, full = _paged_layer(contract) + for _ in range(8): + d = _step(num_tokens, padded=padded) + method.base_method.rows = 0 + out = method.apply(layer=layer, dispatch_output=d).hidden_states + torch.testing.assert_close(out, _reference(full, d)) + for expert, slot in method.policy.resident.items(): + torch.testing.assert_close( + layer.w2_weight[slot], + method.store.host["w2_weight"][expert], + ) + self.assertLess( + method.base_method.rows, 2 * d.topk_output.topk_ids.numel() + ) + + +class TestLoading(CustomTestCase): + def test_rejects_post_load_weight_transform(self): + method, layer, _ = _paged_layer() + method.base_method.process_weights_after_loading = lambda layer: ( + layer.w2_weight.data.mul_(2) + ) + with self.assertRaisesRegex(RuntimeError, "transforms w2_weight"): + method.process_weights_after_loading(layer) + + def test_host_loader_follows_base_method_and_param_tags(self): + """Host rows are loaded as natively: the layer's loader follows the wrapped method's + layout, and sees the parameter's tags (a scale's ``quant_method`` decides its + layout).""" + method, layer, _ = _paged_layer(load=False) + self.assertIs(layer.weight_load_method, method.base_method) + self.assertEqual(method.store.host["w2_weight"].shape, (E, H, H)) + method.store.host["w2_weight"].zero_() + seen = {} + + def impl(self, param, loaded_weight, weight_name, shard_id, expert_id): + seen["tag"] = param.quant_method + param.data[expert_id].copy_(loaded_weight) + + layer._weight_loader_impl = impl.__get__(layer) + layer.w2_weight.weight_loader( + layer.w2_weight, torch.ones(H, H), "experts.w2_weight", "w2", 9 + ) + + self.assertEqual(seen, {"tag": "block"}) + self.assertTrue( + torch.equal(method.store.host["w2_weight"][9], torch.ones(H, H)) + ) + self.assertFalse(method.store.host["w2_weight"][8].any()) + + def test_rejects_online_weight_updates(self): + from sglang.srt.model_executor.model_runner_components.weight_updater import ( + _unsupported_derived_weight_cache_error, + ) + + _, layer, _ = _paged_layer(load=False) + model = torch.nn.Sequential(layer) + self.assertIn( + "--enable-paged-experts", _unsupported_derived_weight_cache_error(model) + ) + + def test_rejects_undeclared_expert_params(self): + """A per-expert tensor the format does not declare would never reach the slots.""" + with self.assertRaisesRegex(RuntimeError, "w13_weight_scale"): + _paged_layer(extra_params=("w13_weight_scale",), load=False) + + def test_repack_runs_per_chunk_and_keeps_its_output(self): + """A post-load step that repacks each expert (new shape, new name) is applied to every + expert of the host store, K at a time; the slots then hold experts 0..K-1 repacked.""" + + class _Repacking: + def process_weights_after_loading(self, layer): + raw = layer.raw_weight.data + packed = (raw * 10 + 1).reshape(raw.shape[0], 2, -1) # per expert + layer.register_parameter( + "packed_weight", torch.nn.Parameter(packed, requires_grad=False) + ) + + class _RepackFormat(ExpertFormat): + checkpoint_params = ("raw_weight",) + paged_params = ("packed_weight",) + repacks_after_loading = True + + @classmethod + def supports(cls, base_method): + return True + + layer = torch.nn.Module() + layer.register_parameter( + "raw_weight", torch.nn.Parameter(torch.zeros(K, 6), requires_grad=False) + ) + fmt = _RepackFormat() + staged = _CpuStore( + layer=layer, + names=fmt.checkpoint_params, + num_experts=E + 1, # a partial last chunk + pin=False, + ) + staged.host["raw_weight"].copy_(torch.arange((E + 1) * 6.0).reshape(E + 1, 6)) + + with patch( + "sglang.srt.layers.moe.paged_experts.formats.HostExpertStore", _CpuStore + ): + packed = fmt.after_loading( + layer=layer, base_method=_Repacking(), store=staged, num_slots=K + ) + + expected = (staged.host["raw_weight"] * 10 + 1).reshape(E + 1, 2, 3) + self.assertTrue(torch.equal(packed.host["packed_weight"], expected)) + self.assertTrue(torch.equal(layer.packed_weight.data, expected[:K])) + + +class TestResidency(CustomTestCase): + def test_lru_evicts_least_recently_used(self): + policy = LRUPolicy(4) # slots start with experts 0..3 + wave = policy.place([0, 5]) # 0 stays; 5 evicts 1, the least recently used + self.assertEqual(wave.loads, [(5, 1)]) + self.assertEqual(wave.slots, [0, 1]) + wave = policy.place([6, 7]) # evicts 2 and 3; 0 and 5 stay + self.assertEqual(sorted(wave.loads), [(6, 2), (7, 3)]) + self.assertEqual(sorted(policy.resident), [0, 5, 6, 7]) + + def test_resident_wave_loads_nothing(self): + self.assertEqual(LRUPolicy(4).place([3, 1]).loads, []) + + def test_waves_serve_residents_before_evicting_them(self): + policy = LRUPolicy(2) + policy.place([2, 3]) # residents {2, 3} + waves = plan_waves(policy=policy, distinct=[0, 1, 2, 3]) + self.assertEqual([w.experts for w in waves], [[2, 3], [0, 1]]) + self.assertEqual(sum(len(w.loads) for w in waves), 2) + + def test_waves_cover_every_expert_once(self): + for distinct, sizes in (([], [0]), ([2, 9], [2]), (list(range(10)), [4, 4, 2])): + with self.subTest(distinct=distinct): + waves = plan_waves(policy=LRUPolicy(4), distinct=distinct) + self.assertEqual([len(w.experts) for w in waves], sizes) + self.assertEqual([e for w in waves for e in w.experts], distinct) + for w in waves: # within a wave, every expert has its own slot + self.assertEqual(len(set(w.slots)), len(w.slots)) + + def test_device_state_round_trips_through_the_policy(self): + """Host-planned steps between graph replays continue from the device's LRU state and + leave theirs for the next replay.""" + state = DeviceResidency(num_experts=E, num_slots=4, device="cpu") + state.slot_expert.copy_(torch.tensor([9, 2, 7, 5], dtype=torch.int32)) + state.slot_lastuse.copy_(torch.tensor([6, 3, 8, 3], dtype=torch.int32)) + state.step.fill_(8) + policy = LRUPolicy(4) + + state.load_into(policy) + self.assertEqual( + list(policy.resident.items()), [(2, 1), (5, 3), (9, 0), (7, 2)] + ) + policy.place([11]) # evicts 2, the least recently used + state.store_from(policy) + + self.assertEqual(state.slot_expert.tolist(), [9, 11, 7, 5]) + self.assertEqual(state.slot_lastuse.tolist(), [10, 12, 11, 9]) + self.assertEqual(int(state.step), 12) + self.assertEqual(state.expert_slot[[9, 11, 7, 5, 2]].tolist(), [0, 1, 2, 3, -1]) + + +class TestDecodeCaptureCap(CustomTestCase): + def _capped(self, bs, max_bs): + decode = SimpleNamespace(bs=list(bs), max_bs=max(bs)) + exec_ctx = SimpleNamespace( + graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode)) + ) + with patch( + "sglang.srt.layers.moe.paged_experts.method.get_exec", return_value=exec_ctx + ): + _cap_decode_capture_batch_sizes(max_bs) + return decode.bs, decode.max_bs + + def test_only_shrinks_the_configured_sizes(self): + self.assertEqual(self._capped([1, 2, 4], 16), ([1, 2, 4], 4)) + self.assertEqual(self._capped([1, 2, 4, 8], 3), ([1, 2, 3], 3)) + + +class TestCompat(CustomTestCase): + def _check(self, num_resident=K, speculative_algorithm=None): + method = "sglang.srt.layers.moe.paged_experts.method." + moe = SimpleNamespace( + enable_eplb=False, + paged_experts_num_resident=num_resident, + enable_fused_moe_sum_all_reduce=False, + ) + layer = SimpleNamespace(top_k=TOP_K, num_fused_shared_experts=0) + with ( + patch( + method + "get_parallel", + return_value=SimpleNamespace( + tp_size=1, ep_size=1, pp_size=1, dp_size=1 + ), + ), + patch(method + "get_exec", return_value=SimpleNamespace(moe=moe)), + patch( + method + "get_spec", + return_value=SimpleNamespace( + speculative_algorithm=speculative_algorithm + ), + ), + patch( + method + "get_model", return_value=SimpleNamespace(load_format="auto") + ), + patch(method + "check_cuda_graph_backend", return_value=True), + ): + check_paged_experts_compat(layer=layer, expert_format=object()) + + def test_accepts_a_supported_config(self): + self._check() + + def test_rejects_speculative_decoding(self): + with self.assertRaisesRegex(RuntimeError, "speculative decoding"): + self._check(speculative_algorithm="EAGLE") + + def test_requires_num_resident(self): + with self.assertRaisesRegex(RuntimeError, "--paged-experts-num-resident"): + self._check(num_resident=None) + + +if __name__ == "__main__": + unittest.main()