[Feat] paged-experts: Serve MoE models whose weights do not fit VRAM (#42557)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
Xiaoyu Zhang
parent
806e7423af
commit
8e7601ae4a
@@ -0,0 +1,212 @@
|
||||
// Paged experts (srt/layers/moe/paged_experts): the residency decision and the expert page-in on
|
||||
// the GPU, so a decode step pages experts without a host sync and can be captured in a CUDA graph.
|
||||
//
|
||||
// decide: for a step whose routed (token, expert) entries fit the K slots, keep every resident
|
||||
// expert the step uses, give each missing expert the least recently used slot the step does not
|
||||
// need, and write the page-in plan (src expert -> dst slot, count) to device buffers.
|
||||
// gather: copy the planned experts from the pinned host store into their slots, for every paged
|
||||
// tensor of a layer in one launch. The count is read on the device, so a replayed graph moves
|
||||
// exactly the experts its decide chose.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <sgl_kernel/tensor.h>
|
||||
#include <sgl_kernel/utils.h>
|
||||
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
namespace {
|
||||
|
||||
// One warp. The evictions depend on each other and are committed one at a time by lane 0; each
|
||||
// victim search and the per-entry bookkeeping are spread over the warp.
|
||||
__global__ void paged_experts_decide_kernel(
|
||||
const int32_t* topk, // [T] routed expert ids of the step (negative: padding)
|
||||
int T,
|
||||
int E,
|
||||
int K,
|
||||
int32_t* step, // [1] step counter, advanced on the device so replays age the LRU
|
||||
int32_t* slot_expert, // [K] expert in each slot (-1: empty)
|
||||
int32_t* expert_slot, // [E] slot of each expert (-1: not resident)
|
||||
int32_t* slot_lastuse, // [K] step each slot was last used
|
||||
int32_t* src, // [K] out: experts to page in
|
||||
int32_t* dst, // [K] out: their slots
|
||||
int32_t* count) { // [1] out: number of page-ins
|
||||
const int lane = threadIdx.x;
|
||||
int now = 0;
|
||||
if (lane == 0) now = ++(*step);
|
||||
now = __shfl_sync(0xffffffff, now, 0);
|
||||
for (int i = lane; i < T; i += 32) {
|
||||
const int e = topk[i];
|
||||
if (e < 0 || e >= E) continue;
|
||||
const int s = expert_slot[e];
|
||||
if (s >= 0) slot_lastuse[s] = now;
|
||||
}
|
||||
__syncwarp();
|
||||
int n = 0;
|
||||
for (int i = 0; i < T; ++i) {
|
||||
const int e = topk[i];
|
||||
if (e < 0 || e >= E || expert_slot[e] >= 0) continue; // the same for every lane
|
||||
// Victim: the least recently used slot holding no expert of this step, lowest slot on
|
||||
// ties. Every slot used this step carries `now`, so one is always left while T <= K.
|
||||
int oldest = INT_MAX, victim = K;
|
||||
for (int s = lane; s < K; s += 32) {
|
||||
const int lastuse = slot_lastuse[s];
|
||||
if (lastuse != now && lastuse < oldest) {
|
||||
oldest = lastuse;
|
||||
victim = s;
|
||||
}
|
||||
}
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
const int other_oldest = __shfl_down_sync(0xffffffff, oldest, offset);
|
||||
const int other_victim = __shfl_down_sync(0xffffffff, victim, offset);
|
||||
if (other_oldest < oldest || (other_oldest == oldest && other_victim < victim)) {
|
||||
oldest = other_oldest;
|
||||
victim = other_victim;
|
||||
}
|
||||
}
|
||||
victim = __shfl_sync(0xffffffff, victim, 0);
|
||||
if (victim == K) continue;
|
||||
if (lane == 0) {
|
||||
const int old = slot_expert[victim];
|
||||
if (old >= 0) expert_slot[old] = -1;
|
||||
slot_expert[victim] = e;
|
||||
expert_slot[e] = victim;
|
||||
slot_lastuse[victim] = now;
|
||||
src[n] = e;
|
||||
dst[n] = victim;
|
||||
}
|
||||
++n;
|
||||
__syncwarp(); // the next search reads this eviction
|
||||
}
|
||||
if (lane == 0) *count = n;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__device__ void
|
||||
copy_rows(const T* host, T* gpu, long row, int n, const int32_t* src, const int32_t* dst, long tid, long stride) {
|
||||
for (long j = tid; j < n * row; j += stride) {
|
||||
const long i = j / row, off = j % row;
|
||||
gpu[dst[i] * row + off] = host[src[i] * row + off];
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void paged_experts_gather_kernel(
|
||||
const int64_t* hosts, // [P] UVA device pointer of each paged tensor's host store
|
||||
const int64_t* gpus, // [P] each paged tensor's K-slot GPU table
|
||||
const int64_t* words, // [P] 4-byte words per expert of each tensor
|
||||
int P,
|
||||
const int32_t* src,
|
||||
const int32_t* dst,
|
||||
const int32_t* count) {
|
||||
const int n = *count;
|
||||
const long stride = static_cast<long>(gridDim.x) * blockDim.x;
|
||||
const long tid = static_cast<long>(blockIdx.x) * blockDim.x + threadIdx.x;
|
||||
for (int p = 0; p < P; ++p) {
|
||||
const long row = words[p];
|
||||
if (row % 4 == 0) { // 16-byte rows: copy in float4, which the host link needs
|
||||
copy_rows(
|
||||
reinterpret_cast<const float4*>(hosts[p]),
|
||||
reinterpret_cast<float4*>(gpus[p]),
|
||||
row / 4,
|
||||
n,
|
||||
src,
|
||||
dst,
|
||||
tid,
|
||||
stride);
|
||||
} else { // small per-expert scalars
|
||||
copy_rows(
|
||||
reinterpret_cast<const int32_t*>(hosts[p]),
|
||||
reinterpret_cast<int32_t*>(gpus[p]),
|
||||
row,
|
||||
n,
|
||||
src,
|
||||
dst,
|
||||
tid,
|
||||
stride);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void paged_experts_decide(
|
||||
tvm::ffi::TensorView topk,
|
||||
tvm::ffi::TensorView step,
|
||||
tvm::ffi::TensorView slot_expert,
|
||||
tvm::ffi::TensorView expert_slot,
|
||||
tvm::ffi::TensorView slot_lastuse,
|
||||
tvm::ffi::TensorView src,
|
||||
tvm::ffi::TensorView dst,
|
||||
tvm::ffi::TensorView count) {
|
||||
using namespace sglang::host;
|
||||
|
||||
SymbolicSize E = {"num_experts"}, K = {"num_slots"}, T = {"num_entries"}, One = {"one"};
|
||||
SymbolicDevice device_;
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({E}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(expert_slot);
|
||||
TensorMatcher({K})
|
||||
.with_dtype<int32_t>()
|
||||
.with_device<kDLCUDA>(device_)
|
||||
.verify(slot_expert)
|
||||
.verify(slot_lastuse)
|
||||
.verify(src)
|
||||
.verify(dst);
|
||||
TensorMatcher({T}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(topk);
|
||||
TensorMatcher({One}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(step).verify(count);
|
||||
RuntimeCheck(T.unwrap() <= K.unwrap(), "paged_experts_decide needs at most K routed entries, got ", T.unwrap());
|
||||
|
||||
LaunchKernel(1, 32, device_.unwrap())(
|
||||
paged_experts_decide_kernel,
|
||||
static_cast<const int32_t*>(topk.data_ptr()),
|
||||
static_cast<int>(T.unwrap()),
|
||||
static_cast<int>(E.unwrap()),
|
||||
static_cast<int>(K.unwrap()),
|
||||
static_cast<int32_t*>(step.data_ptr()),
|
||||
static_cast<int32_t*>(slot_expert.data_ptr()),
|
||||
static_cast<int32_t*>(expert_slot.data_ptr()),
|
||||
static_cast<int32_t*>(slot_lastuse.data_ptr()),
|
||||
static_cast<int32_t*>(src.data_ptr()),
|
||||
static_cast<int32_t*>(dst.data_ptr()),
|
||||
static_cast<int32_t*>(count.data_ptr()));
|
||||
}
|
||||
|
||||
void paged_experts_gather(
|
||||
tvm::ffi::TensorView hosts,
|
||||
tvm::ffi::TensorView gpus,
|
||||
tvm::ffi::TensorView words,
|
||||
tvm::ffi::TensorView src,
|
||||
tvm::ffi::TensorView dst,
|
||||
tvm::ffi::TensorView count) {
|
||||
using namespace sglang::host;
|
||||
|
||||
SymbolicSize P = {"num_tensors"}, K = {"num_slots"}, One = {"one"};
|
||||
SymbolicDevice device_;
|
||||
device_.set_options<kDLCUDA>();
|
||||
TensorMatcher({P}).with_dtype<int64_t>().with_device<kDLCUDA>(device_).verify(hosts).verify(gpus).verify(words);
|
||||
TensorMatcher({K}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(src).verify(dst);
|
||||
TensorMatcher({One}).with_dtype<int32_t>().with_device<kDLCUDA>(device_).verify(count);
|
||||
|
||||
// Decode pages few experts per step, so launch cost matters more than link saturation: a
|
||||
// 2048x512 grid saturates the link better but measured 5% slower end to end (Qwen3-30B, K=90).
|
||||
// An empty plan exits at once.
|
||||
LaunchKernel(256, 256, device_.unwrap())(
|
||||
paged_experts_gather_kernel,
|
||||
static_cast<const int64_t*>(hosts.data_ptr()),
|
||||
static_cast<const int64_t*>(gpus.data_ptr()),
|
||||
static_cast<const int64_t*>(words.data_ptr()),
|
||||
static_cast<int>(P.unwrap()),
|
||||
static_cast<const int32_t*>(src.data_ptr()),
|
||||
static_cast<const int32_t*>(dst.data_ptr()),
|
||||
static_cast<const int32_t*>(count.data_ptr()));
|
||||
}
|
||||
|
||||
int64_t paged_experts_host_device_pointer(tvm::ffi::TensorView pinned) {
|
||||
void* ptr = nullptr;
|
||||
const cudaError_t err = cudaHostGetDevicePointer(&ptr, pinned.data_ptr(), 0);
|
||||
sglang::host::RuntimeCheck(err == cudaSuccess, "cudaHostGetDevicePointer failed: ", cudaGetErrorString(err));
|
||||
return reinterpret_cast<int64_t>(ptr);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Paged-experts residency kernels: the per-step decision and the expert page-in on the GPU.
|
||||
|
||||
Used by ``sglang.srt.layers.moe.paged_experts`` so a decode step pages experts without a host
|
||||
sync and can be captured in a CUDA graph. All index tensors are int32 on the GPU.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_module() -> Module:
|
||||
return load_jit(
|
||||
"paged_experts",
|
||||
cuda_files=["moe/paged_experts.cuh"],
|
||||
cuda_wrappers=[
|
||||
("paged_experts_decide", "paged_experts_decide"),
|
||||
("paged_experts_gather", "paged_experts_gather"),
|
||||
(
|
||||
"paged_experts_host_device_pointer",
|
||||
"paged_experts_host_device_pointer",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def paged_experts_decide(
|
||||
topk: torch.Tensor,
|
||||
step: torch.Tensor,
|
||||
slot_expert: torch.Tensor,
|
||||
expert_slot: torch.Tensor,
|
||||
slot_lastuse: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
count: torch.Tensor,
|
||||
) -> None:
|
||||
"""Make every expert in ``topk`` (``[T]`` routed ids, negative = padding, ``T <= K``) resident.
|
||||
|
||||
Keeps the resident ones, gives each missing one the least recently used slot the step does
|
||||
not need, and updates the residency state in place: ``step`` ``[1]``, ``slot_expert`` and
|
||||
``slot_lastuse`` ``[K]``, ``expert_slot`` ``[E]``. Writes the page-in plan to ``src``/``dst``
|
||||
``[K]`` (expert -> slot) and its length to ``count`` ``[1]``.
|
||||
"""
|
||||
_jit_module().paged_experts_decide(
|
||||
topk, step, slot_expert, expert_slot, slot_lastuse, src, dst, count
|
||||
)
|
||||
|
||||
|
||||
def paged_experts_gather(
|
||||
hosts: torch.Tensor,
|
||||
gpus: torch.Tensor,
|
||||
words: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
dst: torch.Tensor,
|
||||
count: torch.Tensor,
|
||||
) -> None:
|
||||
"""Copy experts ``src[i]`` into slots ``dst[i]`` for ``i < count[0]``, for every paged tensor.
|
||||
|
||||
``hosts``/``gpus``/``words`` are int64 GPU tensors with one entry per paged tensor: the UVA
|
||||
device pointer of its pinned host store (``paged_experts_host_device_pointer``), the address of
|
||||
its GPU table, and its size per expert in 4-byte words. Rows of a multiple of 16 bytes are
|
||||
copied in 16-byte loads.
|
||||
"""
|
||||
_jit_module().paged_experts_gather(hosts, gpus, words, src, dst, count)
|
||||
|
||||
|
||||
def paged_experts_host_device_pointer(pinned: torch.Tensor) -> int:
|
||||
"""The address under which the GPU reads the pinned host tensor ``pinned``."""
|
||||
return int(_jit_module().paged_experts_host_device_pointer(pinned))
|
||||
@@ -955,6 +955,22 @@ class ExecMoe(msgspec.Struct):
|
||||
"Enforce shared experts fusion even when it would normally be disabled (e.g. under DeepEP). Mutually exclusive with --disable-shared-experts-fusion.",
|
||||
] = False
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Paged experts
|
||||
# -------------------------------------------------------------------------
|
||||
enable_paged_experts: A[
|
||||
bool,
|
||||
"Keep only --paged-experts-num-resident routed experts per MoE layer on the GPU and page "
|
||||
"the rest from pinned host memory on demand. Unquantized, block-FP8 or GPTQ experts, "
|
||||
"single GPU. Decode CUDA graphs (full backend) are captured up to K // top_k requests; "
|
||||
"prefill graphs must be breakable or disabled.",
|
||||
] = False
|
||||
paged_experts_num_resident: A[
|
||||
Optional[int],
|
||||
"Number of routed experts per MoE layer kept resident on the GPU. Required with "
|
||||
"--enable-paged-experts; at least the router's top-k.",
|
||||
] = None
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Ktransformers/AMX expert parallelism
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@@ -178,6 +178,7 @@ def fused_marlin_moe(
|
||||
gemm1_alpha: Optional[float] = None,
|
||||
activation: str = "silu",
|
||||
is_gated: bool = True,
|
||||
no_combine: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
This function computes a Mixture of Experts (MoE) layer using two sets of
|
||||
@@ -202,6 +203,8 @@ def fused_marlin_moe(
|
||||
- w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1.
|
||||
- w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2.
|
||||
- num_bits (int): The number of bits in expert weights quantization.
|
||||
- no_combine (bool): Return the router-weighted per-(token, expert) outputs,
|
||||
``[num_tokens, topk, hidden]``, instead of their top-k sum.
|
||||
|
||||
Returns:
|
||||
- torch.Tensor: The output tensor after applying the MoE layer.
|
||||
@@ -410,6 +413,9 @@ def fused_marlin_moe(
|
||||
is_zp_float=False,
|
||||
).view(-1, topk, K)
|
||||
|
||||
if no_combine:
|
||||
return intermediate_cache3
|
||||
|
||||
output = zero_copy_context.get_moe_output(hidden_states)
|
||||
if output is None:
|
||||
output = hidden_states if inplace else torch.empty_like(hidden_states)
|
||||
|
||||
@@ -147,6 +147,12 @@ def _maybe_copy_weight_view_before_h2d(
|
||||
return _copy_weight_view_before_h2d(loaded_weight)
|
||||
|
||||
|
||||
def _weight_layout_method(layer):
|
||||
"""The method whose weight layout the loaders follow: the method a wrapper loads for
|
||||
(``weight_load_method``, set by paged experts) or the layer's own quant method."""
|
||||
return getattr(layer, "weight_load_method", None) or layer.quant_method
|
||||
|
||||
|
||||
def _get_deepep_comm_group(a2a_backend):
|
||||
group = get_parallel().tp_group.device_group
|
||||
|
||||
@@ -470,6 +476,9 @@ class FusedMoE(torch.nn.Module):
|
||||
)
|
||||
|
||||
self.quant_method = quant_method
|
||||
# The method whose weight layout the loader follows when quant_method wraps another
|
||||
# one (None: quant_method itself).
|
||||
self.weight_load_method = None
|
||||
server_args = get_server_args()
|
||||
kt_config = create_kt_config_from_server_args(server_args, layer_id)
|
||||
if kt_config is not None:
|
||||
@@ -489,6 +498,12 @@ class FusedMoE(torch.nn.Module):
|
||||
self.use_flashinfer_trtllm_moe,
|
||||
self.use_deep_gemm,
|
||||
)
|
||||
if get_exec().moe.enable_paged_experts:
|
||||
from sglang.srt.layers.moe.paged_experts import make_for_layer
|
||||
|
||||
self.quant_method = make_for_layer(
|
||||
layer=self, base_method=self.quant_method
|
||||
)
|
||||
_validate_hpc_ops_quant_method(self.quant_method)
|
||||
_validate_deepep_v2_quant_method(self.quant_method)
|
||||
if (
|
||||
@@ -754,7 +769,11 @@ class FusedMoE(torch.nn.Module):
|
||||
# w1, gate_proj: Load into first logical weight of w13.
|
||||
# w3, up_proj: Load into second logical weight of w13.
|
||||
# trtllm cutlass kernel assumes differently
|
||||
switch_w13 = getattr(self.quant_method, "load_up_proj_weight_first", False)
|
||||
switch_w13 = getattr(
|
||||
_weight_layout_method(self),
|
||||
"load_up_proj_weight_first",
|
||||
False,
|
||||
)
|
||||
if (
|
||||
(switch_w13 and shard_id == "w1") or (not switch_w13 and shard_id == "w3")
|
||||
) and self.moe_runner_config.is_gated:
|
||||
@@ -1177,7 +1196,7 @@ class FusedMoE(torch.nn.Module):
|
||||
# compressed-tensors checkpoints with packed weights are stored flipped
|
||||
# TODO (mgoin): check self.quant_method.quant_config.quant_format
|
||||
# against known CompressionFormat enum values that have this quality
|
||||
method = self.quant_method
|
||||
method = _weight_layout_method(self)
|
||||
if self.scheme is not None:
|
||||
method = self.scheme
|
||||
if method.__class__.__name__ == "KTEPWrapperMethod":
|
||||
@@ -1412,7 +1431,7 @@ class FusedMoE(torch.nn.Module):
|
||||
) -> None:
|
||||
# Mirror _weight_loader_impl: the trtllm bf16 prep reshapes expert weights
|
||||
# into block layout; hot weight updates must restore canonical shapes first.
|
||||
method = self.quant_method
|
||||
method = _weight_layout_method(self)
|
||||
if isinstance(method, KTEPWrapperMethod):
|
||||
method = method.gpu_method
|
||||
if isinstance(method, UnquantizedFusedMoEMethod):
|
||||
@@ -1441,7 +1460,7 @@ class FusedMoE(torch.nn.Module):
|
||||
# compressed-tensors checkpoints with packed weights are stored flipped
|
||||
# TODO: check self.quant_method.quant_config.quant_format
|
||||
# against known CompressionFormat enum values that have this quality
|
||||
method = self.quant_method
|
||||
method = _weight_layout_method(self)
|
||||
if self.scheme is not None:
|
||||
method = self.scheme
|
||||
if isinstance(method, Fp8MoEMethod) and (
|
||||
|
||||
@@ -53,6 +53,9 @@ class MoeRunnerConfig:
|
||||
apply_router_weight_on_input: bool = False
|
||||
inplace: bool = True
|
||||
no_combine: bool = False
|
||||
# With no_combine: still apply the router weight, leaving only the top-k sum to
|
||||
# the caller (triton and marlin runners).
|
||||
no_combine_keep_router_weight: bool = False
|
||||
routed_scaling_factor: Optional[float] = None
|
||||
gemm1_alpha: Optional[float] = None
|
||||
gemm1_beta: Optional[float] = None
|
||||
|
||||
@@ -149,6 +149,11 @@ def fused_experts_none_to_marlin(
|
||||
f"Unsupported Marlin MoE activation: {runner_config.activation}"
|
||||
)
|
||||
|
||||
# Marlin always applies the router weight inside its down GEMM.
|
||||
assert (
|
||||
not runner_config.no_combine or runner_config.no_combine_keep_router_weight
|
||||
), "Marlin MoE supports no_combine only with no_combine_keep_router_weight"
|
||||
|
||||
# Use a per-call workspace so captured graphs cannot alias Marlin's
|
||||
# inter-block reduction locks and deadlock during capture.
|
||||
workspace = marlin_make_workspace(hidden_states.device, max_blocks_per_sm=4)
|
||||
@@ -205,6 +210,7 @@ def fused_experts_none_to_marlin(
|
||||
gemm1_alpha=runner_config.gemm1_alpha,
|
||||
activation=runner_config.activation,
|
||||
is_gated=runner_config.is_gated,
|
||||
no_combine=runner_config.no_combine,
|
||||
).to(hidden_states.dtype)
|
||||
|
||||
return StandardCombineInput(
|
||||
|
||||
@@ -162,6 +162,7 @@ class TritonRunnerCore(MoeRunnerCore):
|
||||
activation=self.config.activation,
|
||||
is_gated=self.config.is_gated,
|
||||
no_combine=self.config.no_combine,
|
||||
no_combine_keep_router_weight=self.config.no_combine_keep_router_weight,
|
||||
inplace=self.config.inplace,
|
||||
apply_router_weight_on_input=self.config.apply_router_weight_on_input,
|
||||
routed_scaling_factor=self.config.routed_scaling_factor,
|
||||
|
||||
@@ -262,6 +262,7 @@ def outplace_fused_experts(
|
||||
gate_up_interleaved: bool = True,
|
||||
a1_q: Optional[torch.Tensor] = None,
|
||||
fuse_swiglu_interleaved: bool = False,
|
||||
no_combine_keep_router_weight: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return fused_experts_impl(
|
||||
hidden_states,
|
||||
@@ -296,6 +297,7 @@ def outplace_fused_experts(
|
||||
gate_up_interleaved=gate_up_interleaved,
|
||||
a1_q=a1_q,
|
||||
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
|
||||
no_combine_keep_router_weight=no_combine_keep_router_weight,
|
||||
)
|
||||
|
||||
|
||||
@@ -395,6 +397,7 @@ def fused_experts(
|
||||
gate_up_interleaved=moe_runner_config.gate_up_interleaved,
|
||||
a1_q=a1_q,
|
||||
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
|
||||
no_combine_keep_router_weight=moe_runner_config.no_combine_keep_router_weight,
|
||||
)
|
||||
|
||||
|
||||
@@ -556,6 +559,7 @@ def _fused_moe_kernel_sequence(
|
||||
gate_up_interleaved: bool = True,
|
||||
a1_q: Optional[torch.Tensor] = None,
|
||||
fuse_swiglu_interleaved: bool = False,
|
||||
no_combine_keep_router_weight: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Run the MoE kernel/activation/kernel/combine sequence in a single shot.
|
||||
|
||||
@@ -911,7 +915,8 @@ def _fused_moe_kernel_sequence(
|
||||
sorted_token_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_padded,
|
||||
not apply_router_weight_on_input and not no_combine,
|
||||
not apply_router_weight_on_input
|
||||
and (not no_combine or no_combine_keep_router_weight),
|
||||
1,
|
||||
down_config or config,
|
||||
compute_type=compute_type,
|
||||
@@ -935,6 +940,30 @@ def _fused_moe_kernel_sequence(
|
||||
|
||||
del intermediate_cache2
|
||||
|
||||
return moe_combine_topk(
|
||||
intermediate_cache3=intermediate_cache3,
|
||||
out_hidden_states=out_hidden_states,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
no_combine=no_combine,
|
||||
use_fused_moe_sum_all_reduce=use_fused_moe_sum_all_reduce,
|
||||
out_slice=out_slice,
|
||||
_use_intermediate=_use_intermediate,
|
||||
)
|
||||
|
||||
|
||||
def moe_combine_topk(
|
||||
intermediate_cache3: torch.Tensor,
|
||||
out_hidden_states: torch.Tensor,
|
||||
routed_scaling_factor: Optional[float],
|
||||
no_combine: bool = False,
|
||||
use_fused_moe_sum_all_reduce: bool = False,
|
||||
out_slice: Optional[torch.Tensor] = None,
|
||||
_use_intermediate: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Sum the routed-weighted ``[num_tokens, topk, hidden]`` expert outputs into
|
||||
``out_hidden_states`` (the tail of ``_fused_moe_kernel_sequence``)."""
|
||||
num_tokens, topk = intermediate_cache3.shape[:2]
|
||||
|
||||
if routed_scaling_factor is None:
|
||||
routed_scaling_factor = 1.0
|
||||
|
||||
@@ -1057,6 +1086,7 @@ def fused_experts_impl(
|
||||
gate_up_interleaved: bool = True,
|
||||
a1_q: Optional[torch.Tensor] = None,
|
||||
fuse_swiglu_interleaved: bool = False,
|
||||
no_combine_keep_router_weight: bool = False,
|
||||
):
|
||||
padded_size = padding_size
|
||||
if not (use_fp8_w8a8 or use_int8_w8a8) or block_shape is not None or _use_aiter:
|
||||
@@ -1137,6 +1167,7 @@ def fused_experts_impl(
|
||||
gate_up_interleaved=gate_up_interleaved,
|
||||
a1_q=a1_q,
|
||||
fuse_swiglu_interleaved=fuse_swiglu_interleaved,
|
||||
no_combine_keep_router_weight=no_combine_keep_router_weight,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Paged experts: serve an MoE model whose routed experts do not fit in GPU memory.
|
||||
|
||||
Each MoE layer keeps K of its E routed experts in a K-slot GPU table and pages the rest from
|
||||
pinned host memory; outputs are bit-identical to the unpaged layer. See
|
||||
``docs/docs/advanced_features/paged_experts.mdx`` for the design and the execution flow.
|
||||
|
||||
Modules: ``formats`` (per quantization method), ``runners`` (per runner backend), ``store``
|
||||
(host copy and transfers), ``residency`` (slots and waves), ``executor`` (one forward step),
|
||||
``method`` (the ``FusedMoEMethodBase`` wiring them into a ``FusedMoE`` layer).
|
||||
"""
|
||||
|
||||
from sglang.srt.layers.moe.paged_experts.method import (
|
||||
PagedExpertsMoEMethod,
|
||||
make_for_layer,
|
||||
)
|
||||
|
||||
__all__ = ["PagedExpertsMoEMethod", "make_for_layer"]
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Runs a paged forward step: page in, the wave's GEMMs, merge and finish per the runner
|
||||
contract.
|
||||
|
||||
``EagerExecutor`` decides residency on the host and serves any step, in as many waves as it
|
||||
needs. ``DeviceExecutor`` decides and pages on the GPU without a host sync, so CUDA graphs can
|
||||
capture it; it serves the steps whose routed entries fit the K slots (one wave), which covers
|
||||
decode up to ``K // top_k`` requests. Both finish through the same contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.paged_experts.residency import (
|
||||
DeviceResidency,
|
||||
LRUPolicy,
|
||||
plan_waves,
|
||||
)
|
||||
from sglang.srt.layers.moe.paged_experts.runners import RunnerContract, run_on_slots
|
||||
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
|
||||
|
||||
|
||||
class EagerExecutor:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_method,
|
||||
store: HostExpertStore,
|
||||
policy: LRUPolicy,
|
||||
contract: RunnerContract,
|
||||
num_experts: int,
|
||||
routed_scaling_factor: float,
|
||||
device_residency: Optional[DeviceResidency] = None,
|
||||
):
|
||||
self.base_method = base_method
|
||||
self.store = store
|
||||
self.policy = policy
|
||||
self.contract = contract
|
||||
self.num_experts = num_experts
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
# Shared with a DeviceExecutor, whose steps (and graph replays) move it on the GPU.
|
||||
self.device_residency = device_residency
|
||||
|
||||
def run(self, layer, dispatch_output):
|
||||
if self.device_residency is not None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"Paged experts: a captured decode batch routes more entries than the K "
|
||||
"resident slots; capture batch sizes must stay at or below K // top_k"
|
||||
)
|
||||
self.device_residency.load_into(self.policy)
|
||||
out = self._run(layer, dispatch_output)
|
||||
if self.device_residency is not None:
|
||||
self.device_residency.store_from(self.policy)
|
||||
return out
|
||||
|
||||
def _run(self, layer, dispatch_output):
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_ids = topk_output.topk_ids
|
||||
distinct = [e for e in torch.unique(topk_ids).tolist() if e >= 0]
|
||||
merged = None
|
||||
waves = plan_waves(policy=self.policy, distinct=distinct)
|
||||
for wave in waves:
|
||||
self.store.page_in(layer=layer, loads=wave.loads)
|
||||
# Index E (and -1, which wraps to it) stays -1: padded routing entries are masked.
|
||||
to_slot = torch.full((self.num_experts + 1,), -1, dtype=torch.int32)
|
||||
to_slot[wave.experts] = torch.tensor(wave.slots, dtype=torch.int32)
|
||||
slots = to_slot.to(topk_ids.device)[topk_ids]
|
||||
if len(waves) == 1: # only padding is outside the wave
|
||||
merged = run_on_slots(self.base_method, layer, dispatch_output, slots)
|
||||
break
|
||||
merged = self.contract.run_wave(
|
||||
base_method=self.base_method,
|
||||
layer=layer,
|
||||
dispatch_output=dispatch_output,
|
||||
slots=slots,
|
||||
merged=merged,
|
||||
)
|
||||
return StandardCombineInput(
|
||||
hidden_states=self.contract.finish(
|
||||
merged=merged,
|
||||
hidden_states=dispatch_output.hidden_states,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class DeviceExecutor:
|
||||
"""One wave, decided and paged on the GPU: a step with at most K routed entries needs at
|
||||
most K distinct experts, so they all fit the slots at once."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_method,
|
||||
store: HostExpertStore,
|
||||
residency: DeviceResidency,
|
||||
contract: RunnerContract,
|
||||
routed_scaling_factor: float,
|
||||
):
|
||||
self.base_method = base_method
|
||||
self.store = store
|
||||
self.residency = residency
|
||||
self.contract = contract
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
|
||||
def run(self, layer, dispatch_output):
|
||||
from sglang.kernels.ops.moe.paged_experts import paged_experts_decide
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
|
||||
r = self.residency
|
||||
topk_ids = dispatch_output.topk_output.topk_ids
|
||||
paged_experts_decide(
|
||||
topk_ids.flatten().to(torch.int32),
|
||||
r.step,
|
||||
r.slot_expert,
|
||||
r.expert_slot,
|
||||
r.slot_lastuse,
|
||||
r.src,
|
||||
r.dst,
|
||||
r.count,
|
||||
)
|
||||
self.store.gather(src=r.src, dst=r.dst, count=r.count)
|
||||
# Every routed expert is now resident; padding (-1) stays masked.
|
||||
slots = r.expert_slot[topk_ids.clamp(min=0)].masked_fill(topk_ids < 0, -1)
|
||||
partial = run_on_slots(self.base_method, layer, dispatch_output, slots)
|
||||
return StandardCombineInput(
|
||||
hidden_states=self.contract.finish(
|
||||
merged=partial,
|
||||
hidden_states=dispatch_output.hidden_states,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Per-quantization-method expert layout, one ``ExpertFormat`` per fused-MoE method.
|
||||
|
||||
A format answers three questions for the method it wraps:
|
||||
|
||||
* which per-expert tensors the checkpoint loads (``checkpoint_params``) and which are paged at
|
||||
run time (``paged_params``);
|
||||
* what happens after loading, so the host copy holds exactly what the GPU slots would hold
|
||||
(for methods that repack their experts, the repack runs over the host copy K at a time);
|
||||
* which runner contract (``runners.py``) its base method's runner follows.
|
||||
|
||||
Adding a quantization method means adding a subclass to ``FORMATS``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.paged_experts.runners import (
|
||||
MARLIN,
|
||||
TRITON,
|
||||
RunnerContract,
|
||||
)
|
||||
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
|
||||
|
||||
# Prefixes of FusedMoE's per-expert parameters.
|
||||
_EXPERT_PARAM_PREFIXES = ("w13_", "w2_")
|
||||
|
||||
|
||||
class ExpertFormat(ABC):
|
||||
#: Per-expert tensors the base method creates for loading.
|
||||
checkpoint_params: Tuple[str, ...] = ()
|
||||
#: Per-expert tensors paged at run time.
|
||||
paged_params: Tuple[str, ...] = ()
|
||||
#: Whether the base post-load step repacks the experts; it then runs over the host copy K
|
||||
#: experts at a time.
|
||||
repacks_after_loading: bool = False
|
||||
contract: RunnerContract = TRITON
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def supports(cls, base_method) -> bool:
|
||||
"""Whether this format handles ``base_method``."""
|
||||
|
||||
def check_params(self, layer, num_slots: int) -> None:
|
||||
"""Raise unless the layer's per-expert tensors are exactly the declared ones."""
|
||||
per_expert = {
|
||||
name
|
||||
for name, p in layer.named_parameters(recurse=False)
|
||||
if name.startswith(_EXPERT_PARAM_PREFIXES)
|
||||
and p.dim() > 0
|
||||
and p.shape[0] == num_slots
|
||||
}
|
||||
if per_expert != set(self.checkpoint_params):
|
||||
raise RuntimeError(
|
||||
f"Paged experts: {type(self).__name__} expects per-expert parameters "
|
||||
f"{sorted(self.checkpoint_params)}, the layer has {sorted(per_expert)}"
|
||||
)
|
||||
|
||||
def after_loading(self, layer, base_method, store, num_slots: int):
|
||||
"""Run the base post-load step; return the store to page from, whose experts 0..K-1
|
||||
the GPU slots then hold."""
|
||||
if self.repacks_after_loading:
|
||||
return self._repack(
|
||||
layer=layer,
|
||||
base_method=base_method,
|
||||
staged=store,
|
||||
num_slots=num_slots,
|
||||
)
|
||||
for name, host in store.host.items():
|
||||
getattr(layer, name).data.copy_(host[:num_slots])
|
||||
base_method.process_weights_after_loading(layer)
|
||||
# Paging copies host rows verbatim, so the base method must not transform the
|
||||
# loaded weights (e.g. a kernel-specific shuffle or row interleave).
|
||||
for name, host in store.host.items():
|
||||
gpu = getattr(layer, name).data[0]
|
||||
if not torch.equal(gpu, host[0].to(gpu.device)):
|
||||
raise RuntimeError(
|
||||
f"Paged experts: {type(base_method).__name__} transforms {name} "
|
||||
"after loading, which the host store does not replicate"
|
||||
)
|
||||
return store
|
||||
|
||||
def _repack(self, layer, base_method, staged, num_slots: int):
|
||||
"""Run the base post-load step over the staged checkpoint layout, K experts at a time,
|
||||
collecting its per-expert output into a new store. Only valid for steps that treat
|
||||
every expert independently."""
|
||||
num_experts = next(iter(staged.host.values())).shape[0]
|
||||
templates = {name: getattr(layer, name) for name in staged.host}
|
||||
packed = None
|
||||
for start in range(0, num_experts, num_slots):
|
||||
n = min(num_slots, num_experts - start)
|
||||
# Fresh parameters in the checkpoint layout: the step may replace or rewrite them.
|
||||
for name, template in templates.items():
|
||||
fresh = torch.nn.Parameter(
|
||||
torch.empty_like(template.data), requires_grad=False
|
||||
)
|
||||
fresh.__dict__.update(template.__dict__)
|
||||
fresh.data[:n].copy_(staged.host[name][start : start + n])
|
||||
layer.register_parameter(name, fresh)
|
||||
base_method.process_weights_after_loading(layer)
|
||||
if packed is None:
|
||||
packed = HostExpertStore(
|
||||
layer=layer, names=self.paged_params, num_experts=num_experts
|
||||
)
|
||||
for name, host in packed.host.items():
|
||||
host[start : start + n].copy_(getattr(layer, name).data[:n])
|
||||
for name, host in packed.host.items():
|
||||
getattr(layer, name).data.copy_(host[:num_slots])
|
||||
return packed
|
||||
|
||||
|
||||
class UnquantizedFormat(ExpertFormat):
|
||||
checkpoint_params = paged_params = ("w13_weight", "w2_weight")
|
||||
|
||||
@classmethod
|
||||
def supports(cls, base_method) -> bool:
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
|
||||
return isinstance(base_method, UnquantizedFusedMoEMethod)
|
||||
|
||||
|
||||
class Fp8BlockFormat(ExpertFormat):
|
||||
"""FP8 weights with per-block scales (e.g. 128x128) and dynamic activation scales. On the
|
||||
triton runner the post-load step leaves both untouched, so they page as stored."""
|
||||
|
||||
checkpoint_params = paged_params = (
|
||||
"w13_weight",
|
||||
"w2_weight",
|
||||
"w13_weight_scale_inv",
|
||||
"w2_weight_scale_inv",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports(cls, base_method) -> bool:
|
||||
from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod
|
||||
|
||||
return (
|
||||
isinstance(base_method, Fp8MoEMethod)
|
||||
and base_method.quant_config.weight_block_size is not None
|
||||
and base_method.quant_config.is_checkpoint_fp8_serialized
|
||||
and not base_method.use_mxfp8
|
||||
and not base_method.is_fp4_expert
|
||||
)
|
||||
|
||||
|
||||
class GptqMarlinFormat(ExpertFormat):
|
||||
"""GPTQ int4/int8 experts on the Marlin runner. The post-load step sorts the act-order
|
||||
indices and repacks each expert's weights and scales into Marlin's layout."""
|
||||
|
||||
checkpoint_params = (
|
||||
"w13_qweight",
|
||||
"w2_qweight",
|
||||
"w13_scales",
|
||||
"w2_scales",
|
||||
"w13_qzeros",
|
||||
"w2_qzeros",
|
||||
"w13_g_idx",
|
||||
"w2_g_idx",
|
||||
"w13_g_idx_sort_indices",
|
||||
"w2_g_idx_sort_indices",
|
||||
)
|
||||
# GPTQ-Marlin's MoE apply never passes the zero points to the kernel.
|
||||
paged_params = tuple(n for n in checkpoint_params if not n.endswith("_qzeros"))
|
||||
repacks_after_loading = True
|
||||
contract = MARLIN
|
||||
|
||||
@classmethod
|
||||
def supports(cls, base_method) -> bool:
|
||||
from sglang.srt.layers.quantization.gptq.gptq import GPTQMarlinMoEMethod
|
||||
|
||||
return isinstance(base_method, GPTQMarlinMoEMethod)
|
||||
|
||||
|
||||
FORMATS = (UnquantizedFormat, Fp8BlockFormat, GptqMarlinFormat)
|
||||
|
||||
|
||||
def format_for(base_method) -> Optional[ExpertFormat]:
|
||||
for fmt in FORMATS:
|
||||
if fmt.supports(base_method):
|
||||
return fmt()
|
||||
return None
|
||||
@@ -0,0 +1,323 @@
|
||||
"""The fused-MoE method that wires paged experts into a ``FusedMoE`` layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.moe.paged_experts.executor import DeviceExecutor, EagerExecutor
|
||||
from sglang.srt.layers.moe.paged_experts.formats import ExpertFormat, format_for
|
||||
from sglang.srt.layers.moe.paged_experts.residency import DeviceResidency, LRUPolicy
|
||||
from sglang.srt.layers.moe.paged_experts.store import HostExpertStore
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
from sglang.srt.layers.quantization.base_config import FusedMoEMethodBase
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
cuda_graph_fully_disabled,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
process_model_config,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PagedExpertsMoEMethod(FusedMoEMethodBase, torch.nn.Module):
|
||||
"""Wraps a fused-MoE method so it runs over a K-slot GPU table paged from host memory.
|
||||
|
||||
An ``nn.Module`` like the methods it wraps, so it can replace one as a layer's child."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_method,
|
||||
expert_format: ExpertFormat,
|
||||
num_experts: int,
|
||||
num_resident: int,
|
||||
graphs: bool = False,
|
||||
):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.base_method = base_method
|
||||
self.format = expert_format
|
||||
self.num_experts = num_experts
|
||||
self.num_resident = num_resident
|
||||
self.policy = LRUPolicy(num_resident)
|
||||
self.store = None # built once the base method has created the expert tensors
|
||||
self.executor = None # built once the store holds the paged layout
|
||||
# Some steps may run inside a CUDA graph: decide one-wave steps on the device, so
|
||||
# the graph can capture them.
|
||||
self.graphs = graphs
|
||||
self.device_executor = None
|
||||
self.routed_scaling_factor = 1.0
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
params_dtype,
|
||||
**extra,
|
||||
):
|
||||
# The GPU table has K rows. The loader below sends all E experts to the host store,
|
||||
# so the native expert-id remap never sees ids >= K.
|
||||
layer.num_local_experts = self.num_resident
|
||||
layer._num_local_routed = layer._num_global_routed = self.num_resident
|
||||
self.base_method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=self.num_resident,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size_per_partition,
|
||||
params_dtype=params_dtype,
|
||||
**extra,
|
||||
)
|
||||
# The loader follows the base method's weight layout, as it would without paging.
|
||||
layer.weight_load_method = self.base_method
|
||||
layer._derived_weight_cache_error = (
|
||||
"Online weight updates are not supported with --enable-paged-experts: "
|
||||
"experts already on the GPU would keep the old weights."
|
||||
)
|
||||
self.format.check_params(layer=layer, num_slots=self.num_resident)
|
||||
# A store that only stages the checkpoint layout for a repack need not be pinned.
|
||||
self.store = HostExpertStore(
|
||||
layer=layer,
|
||||
names=self.format.checkpoint_params,
|
||||
num_experts=self.num_experts,
|
||||
pin=not self.format.repacks_after_loading,
|
||||
)
|
||||
for name in self.format.checkpoint_params:
|
||||
getattr(layer, name).weight_loader = self._make_host_loader(
|
||||
layer=layer, name=name
|
||||
)
|
||||
|
||||
def _make_host_loader(self, layer, name):
|
||||
def weight_loader(param, loaded_weight, weight_name, shard_id, expert_id):
|
||||
# Reuse the native per-expert loader (sharding, w1/w3 placement, scale layout)
|
||||
# on a one-row view of the host store, so host rows have exactly the native GPU
|
||||
# layout. The row carries the parameter's loader tags (e.g. ``quant_method``).
|
||||
row = torch.nn.Parameter(
|
||||
self.store.host[name][expert_id : expert_id + 1], requires_grad=False
|
||||
)
|
||||
row.__dict__.update(param.__dict__)
|
||||
layer._weight_loader_impl(
|
||||
param=row,
|
||||
loaded_weight=loaded_weight,
|
||||
weight_name=weight_name,
|
||||
shard_id=shard_id,
|
||||
expert_id=0,
|
||||
)
|
||||
|
||||
return weight_loader
|
||||
|
||||
def create_moe_runner(self, layer, moe_runner_config):
|
||||
self.base_method.create_moe_runner(
|
||||
layer=layer,
|
||||
moe_runner_config=self.format.contract.runner_config(
|
||||
moe_runner_config=moe_runner_config, num_slots=self.num_resident
|
||||
),
|
||||
)
|
||||
self.runner = self.base_method.runner
|
||||
self.format.contract.check(base_method=self.base_method)
|
||||
self.routed_scaling_factor = moe_runner_config.routed_scaling_factor or 1.0
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
self.store = self.format.after_loading(
|
||||
layer=layer,
|
||||
base_method=self.base_method,
|
||||
store=self.store,
|
||||
num_slots=self.num_resident,
|
||||
)
|
||||
residency = None
|
||||
if self.graphs:
|
||||
residency = DeviceResidency(
|
||||
num_experts=self.num_experts,
|
||||
num_slots=self.num_resident,
|
||||
device=getattr(layer, self.format.paged_params[0]).device,
|
||||
)
|
||||
self.store.bind_device_gather(layer)
|
||||
self.device_executor = DeviceExecutor(
|
||||
base_method=self.base_method,
|
||||
store=self.store,
|
||||
residency=residency,
|
||||
contract=self.format.contract,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
self.executor = EagerExecutor(
|
||||
base_method=self.base_method,
|
||||
store=self.store,
|
||||
policy=self.policy,
|
||||
contract=self.format.contract,
|
||||
num_experts=self.num_experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
device_residency=residency,
|
||||
)
|
||||
|
||||
def apply(self, layer, dispatch_output):
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
|
||||
# Decided from shapes alone, so a captured graph always takes the same path.
|
||||
topk_output = dispatch_output.topk_output
|
||||
if (
|
||||
self.device_executor is not None
|
||||
and topk_output.topk_ids.numel() <= self.num_resident
|
||||
):
|
||||
return self.device_executor.run(
|
||||
layer=layer, dispatch_output=dispatch_output
|
||||
)
|
||||
if is_in_breakable_cuda_graph():
|
||||
# The host-planned waves run as an eager break between graph segments.
|
||||
assert (
|
||||
TopKOutputChecker.format_is_standard(topk_output)
|
||||
and dispatch_output.hidden_states_scale is None
|
||||
and dispatch_output.hidden_states_pre_quant is None
|
||||
), (
|
||||
"Paged experts: breakable CUDA graphs need standard, unquantized MoE inputs"
|
||||
)
|
||||
out = torch.empty_like(dispatch_output.hidden_states)
|
||||
self._run_eager_into(
|
||||
layer,
|
||||
dispatch_output.hidden_states,
|
||||
topk_output.topk_weights,
|
||||
topk_output.topk_ids,
|
||||
topk_output.router_logits,
|
||||
out,
|
||||
)
|
||||
return StandardCombineInput(hidden_states=out)
|
||||
return self.executor.run(layer=layer, dispatch_output=dispatch_output)
|
||||
|
||||
def _run_eager_into_impl(
|
||||
self, layer, hidden_states, topk_weights, topk_ids, router_logits, out
|
||||
) -> None:
|
||||
# Flat tensors in, output buffer filled in place: a break replays with the tensors it
|
||||
# was captured with, rebuilt into the dispatch output here.
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardDispatchOutput
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput
|
||||
|
||||
dispatch_output = StandardDispatchOutput(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=None,
|
||||
topk_output=StandardTopKOutput(
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
router_logits=router_logits,
|
||||
),
|
||||
)
|
||||
out.copy_(
|
||||
self.executor.run(
|
||||
layer=layer, dispatch_output=dispatch_output
|
||||
).hidden_states
|
||||
)
|
||||
|
||||
_run_eager_into = eager_on_graph(True)(_run_eager_into_impl)
|
||||
|
||||
|
||||
def make_for_layer(layer, base_method) -> PagedExpertsMoEMethod:
|
||||
expert_format = format_for(base_method)
|
||||
check_paged_experts_compat(layer=layer, expert_format=expert_format)
|
||||
num_experts = layer.num_local_experts
|
||||
num_resident = min(get_exec().moe.paged_experts_num_resident, num_experts)
|
||||
if check_cuda_graph_backend(Phase.DECODE, Backend.FULL):
|
||||
# A full decode graph cannot break out for host-planned waves.
|
||||
_cap_decode_capture_batch_sizes(max(1, num_resident // _router_top_k(layer)))
|
||||
return PagedExpertsMoEMethod(
|
||||
base_method=base_method,
|
||||
expert_format=expert_format,
|
||||
num_experts=num_experts,
|
||||
num_resident=num_resident,
|
||||
graphs=not cuda_graph_fully_disabled(),
|
||||
)
|
||||
|
||||
|
||||
def _cap_decode_capture_batch_sizes(max_bs: int) -> None:
|
||||
"""Capture full decode graphs only at batch sizes whose routed entries fit the K slots,
|
||||
the steps the device executor serves; larger batches run without a graph. The cap itself
|
||||
is captured, so batches up to it pad to a graph. Every MoE layer calls this; only the
|
||||
first changes the config."""
|
||||
decode = get_exec().graph.cuda_graph_config.decode
|
||||
if not decode.bs:
|
||||
return
|
||||
cap = min(max_bs, max(decode.bs))
|
||||
capture_bs = sorted({bs for bs in decode.bs if bs <= cap} | {cap})
|
||||
if capture_bs != list(decode.bs):
|
||||
logger.info(
|
||||
"Paged experts: decode CUDA graphs captured at batch sizes %s (at most K // top_k)",
|
||||
capture_bs,
|
||||
)
|
||||
decode.bs = capture_bs
|
||||
decode.max_bs = cap
|
||||
|
||||
|
||||
def check_paged_experts_compat(layer, expert_format) -> None:
|
||||
"""Raise if the server or layer configuration is outside what paged experts supports;
|
||||
``expert_format`` is None when no format handles the layer's quantization method."""
|
||||
|
||||
problems = []
|
||||
parallel = get_parallel()
|
||||
for name in ("tp_size", "ep_size", "pp_size", "dp_size"):
|
||||
if getattr(parallel, name) > 1:
|
||||
problems.append(f"--{name.replace('_', '-')} must be 1 (single GPU only)")
|
||||
if get_exec().moe.enable_eplb:
|
||||
problems.append("--enable-eplb is not supported")
|
||||
if get_spec().speculative_algorithm:
|
||||
problems.append("speculative decoding is not supported")
|
||||
# Host-planned waves run as eager breaks in a breakable prefill graph; a full graph only
|
||||
# captures steps that fit one wave, which decode can be limited to and prefill cannot.
|
||||
if not (
|
||||
check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
|
||||
or check_cuda_graph_backend(Phase.PREFILL, Backend.DISABLED)
|
||||
):
|
||||
problems.append(
|
||||
"prefill CUDA graphs must use the breakable backend or be disabled"
|
||||
)
|
||||
if not (
|
||||
check_cuda_graph_backend(Phase.DECODE, Backend.FULL)
|
||||
or check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED)
|
||||
):
|
||||
problems.append("decode CUDA graphs must use the full backend or be disabled")
|
||||
if get_model().load_format == "dummy":
|
||||
problems.append(
|
||||
"--load-format dummy is not supported: the host store needs real weights"
|
||||
)
|
||||
num_resident = get_exec().moe.paged_experts_num_resident
|
||||
top_k = _router_top_k(layer)
|
||||
if num_resident is None:
|
||||
problems.append("--paged-experts-num-resident is required")
|
||||
elif num_resident < top_k:
|
||||
problems.append(
|
||||
f"--paged-experts-num-resident must be at least the router's top-k ({top_k})"
|
||||
)
|
||||
if expert_format is None:
|
||||
problems.append(
|
||||
"only unquantized (bf16/fp16), block-quantized FP8 and GPTQ experts are "
|
||||
f"supported, got {type(layer.quant_method).__name__}"
|
||||
)
|
||||
if get_exec().moe.enable_fused_moe_sum_all_reduce:
|
||||
problems.append(
|
||||
"--enable-fused-moe-sum-all-reduce is not supported: paged experts sums the "
|
||||
"top-k outputs itself"
|
||||
)
|
||||
if layer.num_fused_shared_experts:
|
||||
problems.append(
|
||||
"fused shared experts are not supported (--disable-shared-experts-fusion)"
|
||||
)
|
||||
if problems:
|
||||
raise RuntimeError("Paged experts: " + "; ".join(problems))
|
||||
|
||||
|
||||
def _router_top_k(layer) -> int:
|
||||
# Models that route through a separate TopK module (e.g. OLMoE) leave FusedMoE's unset.
|
||||
if layer.top_k is not None:
|
||||
return layer.top_k
|
||||
return process_model_config().hf_text_config.num_experts_per_tok
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Which expert sits in which GPU slot, and how a step's experts are split into waves.
|
||||
|
||||
Pure bookkeeping: no expert moves here. The executor pages ``Wave.loads`` into place before it
|
||||
runs the wave. With CUDA graphs, decode steps decide on the device (``DeviceResidency``), and
|
||||
host-planned steps load that state into the policy and store it back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import List, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
|
||||
class Wave(msgspec.Struct, frozen=True):
|
||||
experts: List[int] # logical expert ids served by this wave
|
||||
slots: List[int] # the GPU slot of each expert, same order
|
||||
loads: List[Tuple[int, int]] # (expert, slot) pairs to page in before the wave runs
|
||||
|
||||
|
||||
class LRUPolicy:
|
||||
"""Chooses the slot of each expert of a wave, evicting the least recently used experts
|
||||
outside it. Slots start holding experts 0..K-1."""
|
||||
|
||||
def __init__(self, num_slots: int):
|
||||
self.num_slots = num_slots
|
||||
# Logical expert -> slot for the resident experts, least recently used first.
|
||||
self.resident = OrderedDict((e, e) for e in range(num_slots))
|
||||
|
||||
def place(self, experts: List[int]) -> Wave:
|
||||
"""Make ``experts`` (at most ``num_slots``) resident and return where they are."""
|
||||
for e in experts:
|
||||
if e in self.resident:
|
||||
self.resident.move_to_end(e)
|
||||
loads = []
|
||||
for e in experts:
|
||||
if e not in self.resident:
|
||||
# Every expert of this wave sits at the LRU tail, so the head is never one of them.
|
||||
_, slot = self.resident.popitem(last=False)
|
||||
self.resident[e] = slot
|
||||
loads.append((e, slot))
|
||||
return Wave(
|
||||
experts=experts, slots=[self.resident[e] for e in experts], loads=loads
|
||||
)
|
||||
|
||||
|
||||
def plan_waves(policy: LRUPolicy, distinct: List[int]) -> List[Wave]:
|
||||
"""Split a step's distinct experts into waves of at most ``num_slots`` and place each.
|
||||
|
||||
A step with no routed expert (all padding) still gets one empty wave, so the layer runs.
|
||||
"""
|
||||
k = policy.num_slots
|
||||
ordered = [e for e in distinct if e in policy.resident] + [
|
||||
e for e in distinct if e not in policy.resident
|
||||
]
|
||||
groups = [ordered[i : i + k] for i in range(0, len(ordered), k)] or [[]]
|
||||
return [policy.place(group) for group in groups]
|
||||
|
||||
|
||||
class DeviceResidency:
|
||||
"""The LRU residency state on the GPU, which the decide kernel updates in place, plus the
|
||||
buffers of its page-in plan. Slots start holding experts 0..K-1, like ``LRUPolicy``."""
|
||||
|
||||
def __init__(self, num_experts: int, num_slots: int, device):
|
||||
i32 = dict(dtype=torch.int32, device=device)
|
||||
self.step = torch.zeros(1, **i32)
|
||||
self.slot_expert = torch.arange(num_slots, **i32)
|
||||
self.expert_slot = torch.full((num_experts,), -1, **i32)
|
||||
self.expert_slot[:num_slots] = self.slot_expert
|
||||
self.slot_lastuse = torch.zeros(num_slots, **i32)
|
||||
self.src = torch.zeros(num_slots, **i32)
|
||||
self.dst = torch.zeros(num_slots, **i32)
|
||||
self.count = torch.zeros(1, **i32)
|
||||
|
||||
def load_into(self, policy: LRUPolicy) -> None:
|
||||
"""Make ``policy`` hold this state: the same residents, least recently used first."""
|
||||
experts = self.slot_expert.tolist()
|
||||
lastuse = self.slot_lastuse.tolist()
|
||||
order = sorted(range(len(experts)), key=lambda s: (lastuse[s], s))
|
||||
policy.resident = OrderedDict((experts[s], s) for s in order)
|
||||
|
||||
def store_from(self, policy: LRUPolicy) -> None:
|
||||
"""Make this state hold ``policy``'s, its recency order as the last steps."""
|
||||
num_slots = self.slot_expert.numel()
|
||||
base = int(self.step.item())
|
||||
slot_expert = [0] * num_slots
|
||||
lastuse = [0] * num_slots
|
||||
for rank, (expert, slot) in enumerate(policy.resident.items(), start=1):
|
||||
slot_expert[slot] = expert
|
||||
lastuse[slot] = base + rank
|
||||
self.slot_expert.copy_(torch.tensor(slot_expert, dtype=torch.int32))
|
||||
self.slot_lastuse.copy_(torch.tensor(lastuse, dtype=torch.int32))
|
||||
self.expert_slot.fill_(-1)
|
||||
self.expert_slot[self.slot_expert.long()] = torch.arange(
|
||||
num_slots, dtype=torch.int32, device=self.expert_slot.device
|
||||
)
|
||||
self.step.fill_(base + num_slots)
|
||||
@@ -0,0 +1,146 @@
|
||||
"""What paged experts needs from a fused-MoE runner backend, one ``RunnerContract`` per backend.
|
||||
|
||||
The executor runs a step in waves (one wave when its experts fit the K slots). The runner
|
||||
returns every routed (token, expert) output already multiplied by its router weight
|
||||
(``no_combine``); the contract runs each wave over only the entries whose expert it holds,
|
||||
writes them into one merged buffer, and performs the top-k sum the runner would have performed
|
||||
itself. A quantization format names the contract of the runner its base method uses; supporting
|
||||
a new backend means adding a contract here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def run_on_slots(base_method, layer, dispatch_output, slots, outside: int = 0):
|
||||
"""Run the base method over the K-slot table: routed entry i goes to GPU slot ``slots[i]``;
|
||||
entries at -1 get expert id ``outside`` and router weight 0, so they contribute nothing."""
|
||||
masked = slots < 0
|
||||
topk_output = dispatch_output.topk_output
|
||||
wave_output = dispatch_output._replace(
|
||||
topk_output=topk_output._replace(
|
||||
topk_ids=slots.masked_fill(masked, outside),
|
||||
topk_weights=topk_output.topk_weights.masked_fill(masked, 0),
|
||||
),
|
||||
)
|
||||
return base_method.apply(layer=layer, dispatch_output=wave_output).hidden_states
|
||||
|
||||
|
||||
class RunnerContract:
|
||||
def runner_config(self, moe_runner_config, num_slots: int):
|
||||
"""The base runner's config: K local experts, router-weighted per-expert outputs."""
|
||||
return replace(
|
||||
moe_runner_config,
|
||||
num_local_experts=num_slots,
|
||||
inplace=False,
|
||||
no_combine=True,
|
||||
no_combine_keep_router_weight=True,
|
||||
)
|
||||
|
||||
def check(self, base_method) -> None:
|
||||
"""Raise if the base method's runner cannot honor ``runner_config``."""
|
||||
|
||||
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
|
||||
"""Compute the routed entries of one wave (``slots`` >= 0) into ``merged``, the step's
|
||||
``[tokens, top_k, hidden]`` entries (None before its first wave), and return it."""
|
||||
raise NotImplementedError
|
||||
|
||||
def finish(self, merged, hidden_states, routed_scaling_factor: float):
|
||||
"""The layer output: the top-k sum of the merged entries."""
|
||||
out = torch.empty_like(hidden_states)
|
||||
self._sum_top_k(merged, out, routed_scaling_factor)
|
||||
return out
|
||||
|
||||
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
|
||||
"""Sum ``[tokens, top_k, hidden]`` into ``out`` exactly as the runner would."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class TritonContract(RunnerContract):
|
||||
def check(self, base_method) -> None:
|
||||
backend = base_method.runner.runner_backend
|
||||
if not backend.is_triton():
|
||||
raise RuntimeError(
|
||||
f"Paged experts: requires the triton MoE runner, got {backend.value}"
|
||||
)
|
||||
|
||||
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
|
||||
config = base_method.runner.config
|
||||
skips = (
|
||||
config.num_experts is None or config.num_experts != config.num_local_experts
|
||||
)
|
||||
partial = run_on_slots(
|
||||
base_method, layer, dispatch_output, slots, outside=-1 if skips else 0
|
||||
)
|
||||
if merged is None:
|
||||
return partial
|
||||
outside = (slots < 0).unsqueeze(-1)
|
||||
return torch.where(outside, merged, partial, out=merged)
|
||||
|
||||
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import (
|
||||
moe_combine_topk,
|
||||
)
|
||||
|
||||
moe_combine_topk(
|
||||
intermediate_cache3=entries,
|
||||
out_hidden_states=out,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
class MarlinContract(RunnerContract):
|
||||
# The Marlin MoE methods build a Marlin runner unconditionally, so there is nothing to check.
|
||||
|
||||
def run_wave(self, base_method, layer, dispatch_output, slots, merged):
|
||||
# Only the wave's entries, compacted: the runner sees no masked rows.
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
assert (
|
||||
dispatch_output.hidden_states_scale is None
|
||||
and dispatch_output.hidden_states_pre_quant is None
|
||||
), "Paged experts: compacted waves need unquantized MoE inputs"
|
||||
if merged is None:
|
||||
merged = hidden_states.new_zeros(*slots.shape, hidden_states.shape[-1])
|
||||
entries = (slots >= 0).flatten().nonzero().squeeze(1)
|
||||
num = entries.numel()
|
||||
if num == 0:
|
||||
return merged
|
||||
# Padded to a few sizes per power of two, so the runner sees few distinct batch
|
||||
# shapes; padding rows go to slot 0 with weight 0 and are dropped.
|
||||
step = max(16, 1 << max(0, num.bit_length() - 4))
|
||||
pad = -num % step
|
||||
tokens = F.pad(entries // slots.shape[1], (0, pad))
|
||||
topk = dispatch_output.topk_output
|
||||
compact = dispatch_output._replace(
|
||||
hidden_states=hidden_states[tokens],
|
||||
topk_output=topk._replace(
|
||||
topk_ids=F.pad(slots.flatten()[entries], (0, pad)).unsqueeze(1),
|
||||
topk_weights=F.pad(
|
||||
topk.topk_weights.flatten()[entries], (0, pad)
|
||||
).unsqueeze(1),
|
||||
router_logits=(
|
||||
None if topk.router_logits is None else topk.router_logits[tokens]
|
||||
),
|
||||
),
|
||||
)
|
||||
out = base_method.apply(layer=layer, dispatch_output=compact).hidden_states
|
||||
merged.view(-1, merged.shape[-1])[entries] = out.view(-1, merged.shape[-1])[
|
||||
:num
|
||||
]
|
||||
return merged
|
||||
|
||||
def _sum_top_k(self, entries, out, routed_scaling_factor: float) -> None:
|
||||
# The reduce fused_marlin_moe itself ends with.
|
||||
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import (
|
||||
moe_sum_reduce,
|
||||
)
|
||||
|
||||
moe_sum_reduce(entries, out, routed_scaling_factor)
|
||||
|
||||
|
||||
TRITON = TritonContract()
|
||||
MARLIN = MarlinContract()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""The host copy of every expert and the transfer of experts into GPU slots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class HostExpertStore:
|
||||
"""Holds all E experts of each paged tensor on the host, in the layer's GPU layout.
|
||||
Page-locked (``pin``) when it serves paging; a store only staged for a post-load repack
|
||||
stays pageable. ``page_in`` is planned on the host; ``gather`` reads a plan on the GPU.
|
||||
|
||||
The GPU tensors are looked up on the layer at every transfer, because a quantization
|
||||
method may rebind them after loading.
|
||||
"""
|
||||
|
||||
def __init__(self, layer, names: Sequence[str], num_experts: int, pin: bool = True):
|
||||
self.pin = pin
|
||||
self.host: Dict[str, torch.Tensor] = {
|
||||
name: self._alloc(
|
||||
shape=(num_experts, *getattr(layer, name).shape[1:]),
|
||||
dtype=getattr(layer, name).dtype,
|
||||
)
|
||||
for name in names
|
||||
}
|
||||
|
||||
def _alloc(self, shape, dtype: torch.dtype) -> torch.Tensor:
|
||||
return torch.empty(shape, dtype=dtype, device="cpu", pin_memory=self.pin)
|
||||
|
||||
def page_in(self, layer, loads: Sequence[Tuple[int, int]]) -> None:
|
||||
"""Copy each ``(expert, slot)`` pair's expert, of every paged tensor, into its slot."""
|
||||
for name, host in self.host.items():
|
||||
gpu = getattr(layer, name).data
|
||||
for expert, slot in loads:
|
||||
gpu[slot].copy_(host[expert], non_blocking=True)
|
||||
|
||||
def bind_device_gather(self, layer) -> None:
|
||||
"""Prepare ``gather``; called once the layer's tensors are final. The gather kernel
|
||||
reads the pinned store directly, through its UVA address."""
|
||||
from sglang.kernels.ops.moe.paged_experts import (
|
||||
paged_experts_host_device_pointer,
|
||||
)
|
||||
|
||||
hosts, gpus, words = [], [], []
|
||||
for name, host in self.host.items():
|
||||
gpu = getattr(layer, name).data
|
||||
row_bytes = host[0].numel() * host.element_size()
|
||||
if row_bytes == 0:
|
||||
continue # e.g. GPTQ without act-order keeps empty g_idx tensors
|
||||
if not (self.pin and gpu.is_contiguous() and row_bytes % 4 == 0):
|
||||
raise RuntimeError(
|
||||
f"Paged experts: CUDA graphs need pinned, contiguous expert rows of whole "
|
||||
f"4-byte words; {name} has {row_bytes} bytes per expert, "
|
||||
f"pinned={host.is_pinned()}, contiguous={gpu.is_contiguous()}"
|
||||
)
|
||||
hosts.append(paged_experts_host_device_pointer(host))
|
||||
gpus.append(gpu.data_ptr())
|
||||
words.append(row_bytes // 4)
|
||||
self._gather_args = [
|
||||
torch.tensor(v, dtype=torch.int64, device=gpu.device)
|
||||
for v in (hosts, gpus, words)
|
||||
]
|
||||
|
||||
def gather(self, src: torch.Tensor, dst: torch.Tensor, count: torch.Tensor) -> None:
|
||||
"""Copy expert ``src[i]`` into GPU slot ``dst[i]`` for ``i < count[0]``, with every
|
||||
index read on the GPU (a device-planned step): capturable in a CUDA graph."""
|
||||
from sglang.kernels.ops.moe.paged_experts import paged_experts_gather
|
||||
|
||||
paged_experts_gather(*self._gather_args, src, dst, count)
|
||||
@@ -850,6 +850,11 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
self.alt_stream is not None
|
||||
and get_is_capture_mode()
|
||||
and not torch.compiler.is_compiling()
|
||||
# Paged experts run as an eager break, which must end the segment on the
|
||||
# capture stream.
|
||||
and not (
|
||||
get_exec().moe.enable_paged_experts and is_in_breakable_cuda_graph()
|
||||
)
|
||||
):
|
||||
final_hidden_states, shared_output = self.forward_normal_dual_stream(
|
||||
hidden_states,
|
||||
|
||||
Reference in New Issue
Block a user