[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)

Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
Cheng Wan
2026-10-20 16:26:15 -07:00
committed by GitHub
co-authored by Aditya Suresh
parent 533020b500
commit 299a75ac5d
83 changed files with 3915 additions and 641 deletions
+1 -1
View File
@@ -43,7 +43,7 @@ jobs:
with:
path: python/sglang/kernels/aot/dist/
merge-multiple: true
pattern: wheel-python3.10-cuda*
pattern: wheel-python3.11-cuda*
- uses: ./.github/actions/download-rust-ext
with:
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
m.def("ggml_moe_get_block_size(int type) -> int");
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
/*
* From csrc/mamba
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
const int nrows_y,
const int nrows_dst,
const int top_k) {
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
const int blocks_per_row_x = ncols_x / qk;
const int blocks_per_col_y = nrows_y / QK8_1;
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
}
const int exp_idx = expert_ids[blockIdx.y];
if (exp_idx > 255 || exp_idx < 0) return;
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
if (exp_idx < 0) {
#pragma unroll
for (int j = 0; j < mmq_x; j += nwarps) {
const int col_dst = token_offs[j / nwarps];
if (col_dst >= ncols_dst) continue;
#pragma unroll
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
const auto row_dst = row_dst_0 + threadIdx.x + i;
if (row_dst < nrows_dst) {
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
}
}
}
return;
}
if (exp_idx > 255) return;
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
const block_q8_1* y = (const block_q8_1*)(vy);
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
if (row >= nrows) {
return;
}
if (expert < 0) {
if (threadIdx.x == 0) {
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
}
return;
}
const int blocks_per_row = ncols / qk;
const int blocks_per_warp = vdr * WARP_SIZE / qi;
@@ -383,14 +383,17 @@ __global__ void Marlin(
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
int parallel = num_tokens_past_padded / moe_block_size;
int num_valid_blocks = parallel;
if constexpr (kIsEP) {
for (int i = 0; i < parallel; i++) {
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
}
// Every warp counts the same inactive blocks without a serial per-thread scan.
int num_invalid_blocks = 0;
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
num_invalid_blocks += expert_ids_ptr[i] == -1;
}
int num_invalid_blocks = parallel - num_valid_blocks;
parallel = num_valid_blocks;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
}
parallel -= num_invalid_blocks;
if (parallel == 0) return;
int k_tiles = prob_k / 16 / thread_k_blocks;
int n_tiles = prob_n / 16 / thread_n_blocks;
@@ -518,23 +521,18 @@ __global__ void Marlin(
if (par_id >= parallel) return;
old_expert_id = expert_id;
if constexpr (kIsEP) {
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
skip_count--;
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
}
} else {
block_id = par_id;
expert_id = expert_ids_ptr[block_id];
skip_count--;
};
}
} else {
block_id = par_id;
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
bool is_ep) {
int tid = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
int32_t neg_count = 0;
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
// Padding and nonlocal routes are both inactive, independently of EP.
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
for (int e = tid; e <= num_experts; e += stride) {
int32_t offset;
+10
View File
@@ -60,6 +60,16 @@ register_kernel(
)
register_kernel(
KernelSpec(
op="moe.shuffle_weighted_sum",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
capabilities=CUDA_ONLY,
)
)
def moe_align_block_size(
topk_ids: torch.Tensor,
num_experts: int,
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
use_int4_w4a16: tl.constexpr,
use_int8_w8a16: tl.constexpr,
even_Ks: tl.constexpr,
filter_expert: tl.constexpr,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
token_mask = offs_token < num_valid_tokens
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
if filter_expert and off_experts == -1:
# -----------------------------------------------------------
# Write back zeros to the output when the expert is not
# in the current expert parallel rank.
if off_experts == -1:
# Padding and nonlocal routes both use -1, including without EP.
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -377,12 +374,10 @@ def fused_moe_kernel(
per_channel_quant: tl.constexpr,
even_Ks: tl.constexpr,
c_sorted: tl.constexpr,
filter_expert: tl.constexpr,
swap_ab: tl.constexpr,
FUSE_ADD_TO_OUTPUT: tl.constexpr,
MASK_OUTPUT: tl.constexpr,
FUSE_SUM_ALL_REDUCE: tl.constexpr,
LORA_PRESERVE_BASE: tl.constexpr,
ROUTER_TOPK: tl.constexpr,
FUSE_SWIGLU: tl.constexpr = False,
USE_GDC: tl.constexpr = False,
@@ -450,16 +445,10 @@ def fused_moe_kernel(
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
off_experts = off_experts_i32.to(tl.int64)
if filter_expert and off_experts == -1:
if FUSE_SWIGLU:
# C is the half-width post-activation buffer here. Rows owned by a
# filtered expert are never read (the down-GEMM CTA for this block
# early-exits before loading A), and an N-wide zero store would run
# past the row end into a neighboring token's data.
return
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
# Write zeros only when this kernel owns the full output; the experimental LoRA
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
if off_experts == -1:
# The down GEMM never reads inactive sorted or half-width intermediates.
# Add/reduce outputs already hold their initial value and must stay intact.
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
lora_preserve_base: bool = False,
fuse_swiglu: bool = False,
) -> None:
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
assert topk_weights.stride(1) == 1
assert sorted_token_ids.stride(0) == 1
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
use_int4_w4a16=use_int4_w4a16,
use_int8_w8a16=use_int8_w8a16,
even_Ks=even_Ks,
filter_expert=filter_expert,
**config,
)
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
per_channel_quant=per_channel_quant,
even_Ks=even_Ks,
c_sorted=c_sorted,
filter_expert=filter_expert,
swap_ab=swap_ab,
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
MASK_OUTPUT=mask_output,
LORA_PRESERVE_BASE=lora_preserve_base,
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
ROUTER_TOPK=router_topk,
FUSE_SWIGLU=fuse_swiglu,
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
num_tokens,
stride_m,
has_expert_map: tl.constexpr,
is_ep: tl.constexpr,
has_topk_ids: tl.constexpr,
top_k: tl.constexpr,
size: tl.constexpr,
routed_scaling_factor: tl.constexpr,
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
if routed_scaling_factor != 1.0:
b_val = b_val * routed_scaling_factor
if has_expert_map:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
mapped_id = tl.load(
expert_map_ptr + tl.maximum(id_val, 0),
mask=m_mask & (id_val >= 0),
other=-1,
)
expert_mask = mapped_id >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
other=0.0,
).to(tl.float32)
elif is_ep:
elif has_topk_ids:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = id_val >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
Shape: (num_tokens, top_k).
outputs: Optional pre-allocated output tensor.
Shape: (num_tokens, hidden_size).
topk_ids: Optional indices of the top-k experts. Used when
`expert_map` is provided. Shape: (num_tokens, top_k).
topk_ids: Optional indices of the top-k experts. Negative ids always
skip the corresponding input load. Shape: (num_tokens, top_k).
expert_map: Optional mapping for Expert Parallelism. A value < 0
indicates an invalid token/expert pair that will be skipped.
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
# Keep is_ep as a compatible host argument; route validity follows ids.
num_tokens, top_k, size = inputs.shape
output_shape = (num_tokens, size)
if outputs is None:
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
num_tokens,
top_k * size,
expert_map is not None,
is_ep,
topk_ids is not None,
top_k,
size,
1.0 if routed_scaling_factor is None else routed_scaling_factor,
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
use_fp32_reduce: bool = False,
is_zp_float: bool = False,
) -> torch.Tensor:
"""Project routes, preserving inactive rows in a caller-provided output.
Inactive rows are zero when this function allocates the output.
"""
device = a.device
# Allocate output if not provided
if c_or_none is not None:
c = c_or_none
else:
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
# Early return for zero-size M
if size_m == 0:
@@ -0,0 +1,74 @@
"""Reduce routed rows while skipping inactive permutation entries."""
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
def _shuffle_weighted_sum_kernel(
inputs,
output,
permutation,
weights,
width: tl.constexpr,
rows,
topk: tl.constexpr,
input_stride: tl.constexpr,
output_stride: tl.constexpr,
weight_stride0,
weight_stride1,
BLOCK: tl.constexpr,
):
token = tl.program_id(0)
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
total = tl.full((BLOCK,), 0, tl.float32)
for slot in range(topk):
row = tl.load(permutation + token * topk + slot).to(tl.int64)
active = (row >= 0) & (row < rows)
values = tl.load(
inputs + row * input_stride + columns,
mask=active & (columns < width),
other=0,
).to(tl.float32)
factor = tl.load(
weights + token * weight_stride0 + slot * weight_stride1,
mask=active,
other=0,
).to(tl.float32)
total += values * factor
tl.store(
output + token.to(tl.int64) * output_stride + columns,
total,
mask=columns < width,
)
def shuffle_weighted_sum(
inputs: torch.Tensor,
output: torch.Tensor,
permutation: torch.Tensor,
weights: torch.Tensor,
) -> None:
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
assert inputs.ndim == output.ndim == weights.ndim == 2
assert inputs.shape[1] == output.shape[1]
assert inputs.stride(1) == output.stride(1) == 1
assert weights.shape[0] == output.shape[0]
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
if output.numel() == 0:
return
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
inputs,
output,
permutation,
weights,
output.shape[1],
inputs.shape[0],
weights.shape[1],
inputs.stride(0),
output.stride(0),
weights.stride(0),
weights.stride(1),
BLOCK=256,
)
@@ -125,6 +125,10 @@ class _StageExecutor:
# handling DP attention
forward_batch: ForwardBatch = inputs["forward_batch"]
self._child_ctx = replace(
child_ctx if child_ctx is not None else get_forward_context(),
moe_num_token_non_padded=forward_batch.moe_num_token_non_padded,
)
self._global_dp_buffer_len = forward_batch.global_dp_buffer_len
self._local_dp_buffer_len = forward_batch.tbo_padded_len
self._global_num_tokens = forward_batch.global_num_tokens_cpu
@@ -39,13 +39,16 @@ class ExpertLocationDispatchInfo:
rank_invariant: bool = False
@classmethod
def init_new(cls, layer_id: int):
def init_new(cls, layer_id: int, *, is_draft: bool = False):
if is_draft:
return None
ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm
expert_location_metadata = get_global_expert_location_metadata()
assert expert_location_metadata is not None
if ep_dispatch_algorithm is None:
return None
expert_location_metadata = get_global_expert_location_metadata()
assert expert_location_metadata is not None, (
"Expert dispatch requires placement metadata"
)
return cls(
ep_dispatch_algorithm=ep_dispatch_algorithm,
@@ -1,12 +1,12 @@
from typing import TYPE_CHECKING, Optional
import torch
from sgl_kernel_npu.norm.l1_norm import l1_norm
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
_zero_topk_weights_padded_region,
capture_routed_experts_if_allowed,
select_experts,
)
@@ -59,6 +59,8 @@ def fused_topk_npu(
)
if renormalize:
from sgl_kernel_npu.norm.l1_norm import l1_norm
topk_weights = l1_norm(
topk_weights
if topk_config.num_shared_topk_slots == 0
@@ -109,9 +111,44 @@ def fused_topk_npu(
expert_location_dispatch_info=expert_location_dispatch_info,
)
if expert_location_dispatch_info is not None:
topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info)
get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids)
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
return _finalize_topk_npu(
topk_ids=topk_ids,
topk_weights=topk_weights,
router_logits=router_logits,
topk_config=topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
layer_id=layer_id,
)
def _finalize_topk_npu(
*,
topk_ids,
topk_weights,
router_logits,
topk_config,
num_token_non_padded,
expert_location_dispatch_info,
layer_id,
):
num_shared = topk_config.num_shared_topk_slots
routed_ids = topk_ids[:, :-num_shared] if num_shared else topk_ids
capture_routed_experts_if_allowed(topk_config, layer_id, routed_ids)
routed_ids = topk_ids_logical_to_physical(routed_ids, expert_location_dispatch_info)
topk_ids = (
torch.cat([routed_ids, topk_ids[:, -num_shared:]], dim=-1)
if num_shared
else routed_ids
)
# NPU dispatch retains valid IDs; duplicating a padded row's routes onto
# expert zero can exceed per-expert capacity in fused dispatch backends.
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
recorder_ids = routed_ids
if num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_ids = routed_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
get_global_expert_distribution_recorder().on_select_experts(topk_ids=recorder_ids)
return StandardTopKOutput(topk_weights, topk_ids, router_logits)
@@ -1209,10 +1209,6 @@ class AiterAttnBackend(AttentionBackend):
bs, forward_batch.seq_lens, forward_batch.req_pool_indices, max_kv_len
)
# Build cu_seqlens_q from this batch's extend lengths. The standard
# prefill metadata path leaves self.qo_indptr unset (qo_indptr=None in
# ForwardMetadata), so relying on it corrupts multi-sequence batches
# (only bs=1 happens to work).
cu_seqlens_q = torch.zeros(bs + 1, dtype=torch.int32, device=q.device)
cu_seqlens_q[1:] = torch.cumsum(
forward_batch.extend_seq_lens.to(torch.int32), dim=0
@@ -2598,7 +2594,7 @@ class AiterAttnBackend(AttentionBackend):
self.forward_metadata = ForwardMetadata(
self.indices_updater_prefill.kv_indptr,
self.indices_updater_prefill.kv_indices,
None,
self.indices_updater_prefill.qo_indptr[: bs + 1],
None,
max(forward_batch.extend_seq_lens_cpu),
forward_batch.seq_lens_cpu.max().item(),
@@ -4423,7 +4419,7 @@ class AiterAttnBackend(AttentionBackend):
self.k_scale.reshape(1), # Q is cast at unit scale.
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
@@ -4499,7 +4495,7 @@ class AiterAttnBackend(AttentionBackend):
fp8_q_descale.reshape(1),
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
@@ -4538,8 +4534,8 @@ class AiterAttnBackend(AttentionBackend):
fp8_q_descale.reshape(1),
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
self.forward_metadata.max_q_len,
self.forward_metadata.max_q_len,
softmax_scale=layer.scaling,
@@ -4628,7 +4624,7 @@ class AiterAttnBackend(AttentionBackend):
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_cache,
v_cache,
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
kv_indptr_arg,
page_table,
self.forward_metadata.max_q_len,
@@ -94,13 +94,13 @@ def forward_extend_vectorized_5d(
kv_indices_lin = torch.arange(
total_tokens, dtype=torch.int32, device=k_lin.device
)
kv_indptr_lin = backend.qo_indptr[:bs0]
kv_indptr_lin = backend.forward_metadata.qo_indptr[:bs0]
max_q = int(backend.forward_metadata.max_q_len)
o = mha_batch_prefill_func(
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_lin,
v_lin,
backend.qo_indptr[:bs0],
backend.forward_metadata.qo_indptr[:bs0],
kv_indptr_lin,
kv_indices_lin,
max_q,
@@ -187,7 +187,7 @@ def forward_extend_vectorized_5d(
q_local.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_lin,
v_lin,
backend.qo_indptr[:bs0],
backend.forward_metadata.qo_indptr[:bs0],
kv_indptr_lin,
kv_indices_lin,
max_q,
+5 -4
View File
@@ -10,7 +10,6 @@ from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
if _is_cuda:
from sgl_kernel import (
apply_shuffle_mul_sum,
es_fp8_blockwise_scaled_grouped_mm,
es_sm100_mxfp8_blockscaled_grouped_mm,
es_sm100_mxfp8_blockscaled_grouped_quant,
@@ -23,6 +22,7 @@ if _is_cuda:
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
shuffle_rows_with_scales,
)
from sglang.kernels.ops.moe.shuffle_weighted_sum import shuffle_weighted_sum
def cutlass_fused_experts_fp8(
@@ -142,8 +142,9 @@ def cutlass_fused_experts_fp8(
topk = topk_ids.size(1)
device = a.device
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
# prepare_moe_input writes only live routes; gathers still span the tail.
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
if use_mxfp8:
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
@@ -345,5 +346,5 @@ def cutlass_fused_experts_fp8(
if output is None:
output = torch.empty((m, k), device=device, dtype=out_dtype)
apply_shuffle_mul_sum(c2, output, c_map, topk_weights.to(out_dtype))
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
return output
@@ -5,7 +5,6 @@ from typing import Optional
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda, is_cuda_alike
_is_cuda = is_cuda()
@@ -128,8 +127,8 @@ def cutlass_w4a8_moe(
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
device = a.device
if get_parallel().moe_ep_size > 1:
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
# Both graph padding and nonlocal routes use -1, including without EP.
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
src2dst = cutlass_w4_run_moe_ep_preproess(
topk_ids,
@@ -144,7 +143,11 @@ def cutlass_w4a8_moe(
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
if a1_scale is None:
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
per_tensor_absmax_fp8(a, a1_scale)
# Inactive rows must not change the quantization of live expert inputs.
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
scale_input = a.masked_fill(inactive_rows, 0.0)
per_tensor_absmax_fp8(scale_input, a1_scale)
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
pre_reorder_for_cutlass_moe(
a,
@@ -305,15 +305,14 @@ def fused_marlin_moe(
device=hidden_states.device,
dtype=hidden_states.dtype,
)
# Marlin skips masked expert rows, so their shared cache must start at zero.
intermediate_cache13 = torch.zeros(
(M * topk_ids.shape[1] * max(gemm1_n, K),),
intermediate_cache13 = torch.empty(
(M * topk * max(gemm1_n, K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * gemm1_n]
intermediate_cache1 = intermediate_cache1.view(-1, gemm1_n)
intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K]
intermediate_cache3 = intermediate_cache13[: M * topk * K]
intermediate_cache3 = intermediate_cache3.view(-1, K)
use_atomic_add = (
@@ -381,9 +380,9 @@ def fused_marlin_moe(
else:
raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}")
if expert_map is not None:
intermediate_cache3.zero_()
# Gate-up is dead after activation, so its storage can hold the down output.
# Skipped down routes must start at zero before the top-k reduction.
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
intermediate_cache3,
@@ -74,6 +74,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_global_dwdp_manager,
get_parallel,
get_server_args,
@@ -367,7 +368,8 @@ class FusedMoE(torch.nn.Module):
self.params_dtype = params_dtype
self.layer_name = prefix
self.layer_id = layer_id
self.uses_expert_location = uses_expert_location
self.is_draft = get_flags().moe.in_speculative_scope
self.uses_expert_location = uses_expert_location and not self.is_draft
self.top_k = top_k
self.hidden_size = hidden_size
self.num_experts = num_experts
@@ -1559,6 +1561,14 @@ class FusedMoE(torch.nn.Module):
return forward_fuseep(self, hidden_states, topk_output)
if is_in_tc_piecewise_cuda_graph():
if (
TopKOutputChecker.format_is_bypassed(topk_output)
and topk_output.expert_location_dispatch_info is not None
):
# The opaque logits-only op cannot carry placement metadata.
return self.forward_impl(
hidden_states, topk_output, pre_quant_input=pre_quant_input
)
if TopKOutputChecker.format_is_standard(topk_output):
return moe_forward_piecewise_cuda_graph_impl(
hidden_states,
@@ -1872,7 +1882,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
) -> torch.Tensor:
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=TopKConfig(
top_k=top_k,
topk_group=topk_group,
@@ -1880,6 +1890,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
correction_bias=correction_bias,
renormalize=renormalize,
allow_routed_experts_capture=allow_routed_experts_capture,
layer_id=layer_id,
),
)
forward_context = get_tc_piecewise_forward_context()
+38 -20
View File
@@ -21,11 +21,10 @@ from sglang.srt.layers.moe.topk import (
_mask_topk_ids_padded_region,
_zero_topk_weights_padded_region,
remap_topk_for_per_rank_shared_slots,
resolve_routing_inputs,
)
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
)
from sglang.srt.runtime_context import get_model
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
from sglang.srt.runtime_context import get_flags, get_model
from sglang.srt.utils import is_hip, is_npu, is_xpu
logger = logging.getLogger(__name__)
@@ -46,9 +45,13 @@ class HashTopK(nn.Module):
routed_scaling_factor=1.5,
apply_routed_scaling_factor_on_output=False,
layer_id: Optional[int] = None,
is_draft: Optional[bool] = None,
):
super().__init__()
self.layer_id = layer_id
self.is_draft = (
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
)
self.shared_experts = get_model().shared_experts
self.enable_waterfill = self.shared_experts.use_waterfill
@@ -75,6 +78,9 @@ class HashTopK(nn.Module):
"with fused shared experts; pass --disable-shared-experts-fusion."
)
def routing_inputs(self):
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
@property
def num_shared_topk_slots(self) -> int:
return self.shared_experts.num_shared_topk_slots
@@ -100,7 +106,8 @@ class HashTopK(nn.Module):
self, device: torch.device, *, layer_id: Optional[int] = None
):
topk = self.topk - self.num_shared_topk_slots
if layer_id is not None:
if not self.is_draft:
layer_id = self.layer_id
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
lplb_solver = get_global_lplb_solver(layer_id)
@@ -226,6 +233,12 @@ class HashTopK(nn.Module):
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
):
num_token_non_padded, expert_location_dispatch_info = resolve_routing_inputs(
layer_id=self.layer_id,
is_draft=self.is_draft,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
)
assert input_ids.shape[0] == hidden_states.shape[0] == router_logits.shape[0], (
f"{input_ids.shape=} {hidden_states.shape=} {router_logits.shape=}"
)
@@ -252,6 +265,9 @@ class HashTopK(nn.Module):
topk_weights = topk_weights * self.routed_scaling_factor
num_shared_topk_slots = self.num_shared_topk_slots
routed_ids = (
topk_ids[:, :-num_shared_topk_slots] if num_shared_topk_slots else topk_ids
)
log2phy_prob = None
if (
expert_location_dispatch_info is not None
@@ -263,18 +279,13 @@ class HashTopK(nn.Module):
lplb_solver = get_global_lplb_solver(self.layer_id)
if lplb_solver is not None:
log2phy_prob = lplb_solver.solve(topk_ids)
log2phy_prob = lplb_solver.solve(routed_ids)
recorder_topk_ids = None
routed_cols = topk_ids_logical_to_physical(
routed_ids, expert_location_dispatch_info, log2phy_prob
)
recorder_topk_ids = routed_cols if num_shared_topk_slots else None
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
# The trailing columns are placeholders; the remap appends the real ones.
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots],
expert_location_dispatch_info,
log2phy_prob,
)
recorder_topk_ids = routed_cols
num_physical_routed_experts = (
expert_location_dispatch_info.num_physical_experts
if expert_location_dispatch_info is not None
@@ -292,11 +303,13 @@ class HashTopK(nn.Module):
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
),
)
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
elif num_shared_topk_slots:
topk_ids = torch.cat(
[routed_cols, topk_ids[:, -num_shared_topk_slots:]], dim=-1
)
if is_hip():
else:
topk_ids = routed_cols
if _is_hip or _is_npu:
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
else:
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
@@ -304,6 +317,11 @@ class HashTopK(nn.Module):
_mask_topk_ids_padded_region(recorder_topk_ids, num_token_non_padded)
if recorder_topk_ids is None:
recorder_topk_ids = topk_ids
if _is_npu and num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_topk_ids = recorder_topk_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_topk_ids
)
@@ -311,7 +329,7 @@ class HashTopK(nn.Module):
topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits
)
topk_output = self._apply_waterfill(topk_output, hidden_states.shape[0])
if is_hip():
if _is_hip or _is_npu:
_zero_topk_weights_padded_region(
topk_output.topk_weights, num_token_non_padded
)
-15
View File
@@ -23,7 +23,6 @@ import torch
from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
from sglang.srt.layers.moe.mega_moe_sm90 import (
@@ -263,16 +262,6 @@ def _run_mega_routed(
if num_tokens > 0:
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
num_token_non_padded = (
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
if moe.experts.uses_expert_location
else None
)
if (
not moe.is_hash
and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None
@@ -281,15 +270,11 @@ def _run_mega_routed(
moe,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
)
else:
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
topk_ids = topk_output.topk_ids
@@ -14,7 +14,6 @@ import torch
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
@@ -307,25 +306,16 @@ def _run_mega_routed(
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.num_token_non_padded
if forward_batch is not None
else None
),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
if moe.experts.uses_expert_location
else None
),
**topk_kwargs,
)
x_in = hidden_states
topk_ids = topk_output.topk_ids.to(torch.int32)
topk_weights = topk_output.topk_weights.to(torch.float32)
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
num_token_non_padded, _ = moe.topk.routing_inputs()
if num_token_non_padded is not None:
_fill_padded_rows(
topk_ids,
forward_batch.num_token_non_padded,
num_token_non_padded,
experts,
)
else:
+158 -20
View File
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_forward,
get_lora,
get_model,
@@ -110,6 +111,10 @@ from sglang.srt.layers.moe.utils import (
RoutingMethodType,
has_per_rank_fused_shared_slots,
)
from sglang.srt.model_executor.forward_context import (
get_forward_context,
has_forward_context,
)
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import (
cpu_has_amx_support,
@@ -279,6 +284,8 @@ class TopKConfig:
# Draft-side MoE blocks set this False so they never write the target's
# process-global routed-experts capture buffer.
allow_routed_experts_capture: bool = True
# Lazy materialization still needs the producer's solver and capture slot.
layer_id: Optional[int] = None
_correction_bias_dtype_cache: Optional[torch.Tensor] = field(
default=None, init=False, repr=False, compare=False
)
@@ -393,10 +400,10 @@ class TritonKernelTopKOutput(NamedTuple):
class BypassedTopKOutput(NamedTuple):
"""Bypassed top-k output format."""
"""Lazy routing that can be materialized or consumed by an in-kernel router."""
hidden_states: torch.Tensor
router_logits: torch.Tensor
raw_router_logits: torch.Tensor
topk_config: TopKConfig
num_token_non_padded: Optional[torch.Tensor] = None
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None
@@ -405,14 +412,23 @@ class BypassedTopKOutput(NamedTuple):
def format(self) -> TopKOutputFormat:
return TopKOutputFormat.BYPASSED
@property
def router_logits(self) -> torch.Tensor:
if self.expert_location_dispatch_info is not None:
raise NotImplementedError(
"Expert placement remapping requires materialized TopK routing; "
"this MoE kernel routes directly from logits."
)
return self.raw_router_logits
def to_standard(self, layer_id: Optional[int] = None) -> StandardTopKOutput:
"""Materialize routing tensors. Used by MoE kernels that need explicit
topk_ids / topk_weights rather than doing routing internally."""
return select_experts(
hidden_states=self.hidden_states,
router_logits=self.router_logits,
router_logits=self.raw_router_logits,
topk_config=self.topk_config,
layer_id=layer_id,
layer_id=self.topk_config.layer_id if layer_id is None else layer_id,
num_token_non_padded=self.num_token_non_padded,
expert_location_dispatch_info=self.expert_location_dispatch_info,
)
@@ -630,6 +646,24 @@ def _can_bypass_trtllm_routing(config: TopKConfig) -> bool:
# -------------------------------- TopK ---------------------------------------
def resolve_routing_inputs(
*,
layer_id: int,
is_draft: bool,
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
):
if num_token_non_padded is None and has_forward_context():
get_bound = get_forward_context().moe_num_token_non_padded
if get_bound is not None:
num_token_non_padded = get_bound()
if is_draft:
expert_location_dispatch_info = None
elif expert_location_dispatch_info is None:
expert_location_dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id)
return num_token_non_padded, expert_location_dispatch_info
class TopK(BaseFusedOp):
"""
Parameters:
@@ -649,6 +683,7 @@ class TopK(BaseFusedOp):
top_k: int,
*,
layer_id: int,
is_draft: Optional[bool] = None,
use_grouped_topk: bool = False,
topk_group: Optional[int] = None,
num_expert_group: Optional[int] = None,
@@ -674,6 +709,9 @@ class TopK(BaseFusedOp):
assert num_expert_group is not None and topk_group is not None
self.layer_id = layer_id
self.is_draft = (
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
)
self._runner_backend_bound = False
shared_experts = get_model().shared_experts
@@ -686,6 +724,7 @@ class TopK(BaseFusedOp):
self.is_fp4_experts = is_fp4_experts
self.topk_config = TopKConfig(
top_k=top_k + num_shared_slots,
layer_id=layer_id,
use_grouped_topk=use_grouped_topk,
renormalize=renormalize,
topk_group=topk_group,
@@ -703,7 +742,21 @@ class TopK(BaseFusedOp):
scoring_func=scoring_func,
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
fused_gate_packed_ids=fused_gate_packed_ids,
allow_routed_experts_capture=allow_routed_experts_capture,
allow_routed_experts_capture=allow_routed_experts_capture
and not self.is_draft,
)
def routing_inputs(self):
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
def _resolve_routing_inputs(
self, num_token_non_padded, expert_location_dispatch_info
):
return resolve_routing_inputs(
layer_id=self.layer_id,
is_draft=self.is_draft,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
)
def bind_runner_backend(
@@ -778,6 +831,11 @@ class TopK(BaseFusedOp):
dynamic_expert_bias: Optional[torch.Tensor] = None,
router_logits_partials: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if router_logits_partials is not None:
rocm_router_reduce_partials(router_logits_partials, router_logits)
self.topk_config.torch_native = True
@@ -805,6 +863,12 @@ class TopK(BaseFusedOp):
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
fixed-order sum is the logits; router_logits is then a buffer the fused gate
fills, and any other reader first reduces the partials into it."""
explicit_dispatch_info = expert_location_dispatch_info is not None
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if dynamic_expert_bias is not None:
output_format = TopKOutputFormat.STANDARD
elif self.topk_config.output_format is not None:
@@ -814,6 +878,17 @@ class TopK(BaseFusedOp):
"TopK requires the owning experts' runner backend or an explicit output format."
)
requires_remapping = expert_location_dispatch_info is not None and (
explicit_dispatch_info
or is_expert_remapping_enabled()
or expert_location_dispatch_info.ep_dispatch_algorithm in ("fake", "lp")
)
if output_format == TopKOutputFormat.TRITON_KERNEL and requires_remapping:
raise NotImplementedError(
"Expert placement remapping requires a MoE runner with standard TopK routing; "
f"got {output_format.name}."
)
if (
router_logits_partials is not None
and output_format != TopKOutputFormat.STANDARD
@@ -833,6 +908,7 @@ class TopK(BaseFusedOp):
router_logits,
self.topk_config.top_k,
sm_first=not self.topk_config.renormalize,
n_rows=num_token_non_padded,
)
return TritonKernelTopKOutput(
a_ragged_metadata,
@@ -844,10 +920,12 @@ class TopK(BaseFusedOp):
elif output_format == TopKOutputFormat.BYPASSED:
return BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=self.topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
expert_location_dispatch_info=(
expert_location_dispatch_info if requires_remapping else None
),
)
else:
self.topk_config.torch_native = False
@@ -875,6 +953,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
topk_output = select_experts(
hidden_states=hidden_states,
layer_id=self.layer_id,
@@ -895,6 +978,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if dynamic_expert_bias is not None:
self.topk_config.torch_native = False
return select_experts(
@@ -923,12 +1011,13 @@ class TopK(BaseFusedOp):
) -> TopKOutput:
"""Return an empty topk output for a rank with zero tokens this forward.
When ``layer_id`` is provided and the active dispatch algorithm is LP,
also calls ``LPLBSolver.solve(empty)`` so that this rank participates
in the EP all-reduce. Without this, an empty rank would skip the
collective and deadlock under DP-attention.
Target layers call their owning ``LPLBSolver.solve(empty)`` so this
rank participates in the EP all-reduce even without tokens. Draft
layers skip the target solver. The optional ``layer_id`` is retained
for caller compatibility; the router's owning layer determines the solver.
"""
if layer_id is not None:
if not self.is_draft:
layer_id = self.layer_id
# Skip the full ExpertLocationDispatchInfo allocation — we only
# need the per-layer solver to participate in the EP all-reduce.
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
@@ -972,6 +1061,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
self.topk_config.torch_native = True
# [NOTE] XPU device support for topk kernels
# - support 'topk_softmax' and 'topk_sigmoid'
@@ -1816,6 +1910,12 @@ def _mask_topk_ids_padded_region(
) -> None:
if num_token_non_padded is None:
return
if not topk_ids.is_cuda:
if _is_npu:
return
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
topk_ids.masked_fill_((rows >= num_token_non_padded)[:, None], fill_value)
return
# TODO: let the kernel support other dtypes
if _is_cuda and topk_ids.dtype == torch.int32 and fill_value == -1:
mask_topk_ids(topk_ids, num_token_non_padded)
@@ -1834,6 +1934,10 @@ def _zero_topk_weights_padded_region(
):
if num_token_non_padded is None:
return
if not topk_weights.is_cuda:
rows = torch.arange(topk_weights.shape[0], device=topk_weights.device)
topk_weights.masked_fill_((rows >= num_token_non_padded)[:, None], 0.0)
return
if _can_fuse_padded_region(topk_weights):
_fill_padded_rows(topk_weights, num_token_non_padded, 0.0)
return
@@ -2420,12 +2524,27 @@ def _post_process_topk_ids(
lplb_solver = get_global_lplb_solver(layer_id)
if lplb_solver is not None:
log2phy_prob = lplb_solver.solve(topk_ids)
lp_routed_ids = (
topk_ids[:, :-num_shared_topk_slots]
if router_wrote_shared_cols
else topk_ids
)
log2phy_prob = lplb_solver.solve(lp_routed_ids)
if log2phy_prob is not None:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
)
if router_wrote_shared_cols:
shared_cols = topk_ids[:, -num_shared_topk_slots:]
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots],
expert_location_dispatch_info,
log2phy_prob,
)
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
recorder_topk_ids = topk_ids[:, :-num_shared_topk_slots]
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
)
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
elif router_wrote_shared_cols:
# Shared IDs are outside EPLB's routed-expert table for both global
@@ -2506,6 +2625,21 @@ def _post_process_topk_ids(
# That final pass re-zeros after any shared-expert append/remap, so a
# second zeroing here would be redundant (zeroing is idempotent).
else:
if num_shared_topk_slots > 0:
shared_cols = topk_ids[:, -num_shared_topk_slots:]
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info
)
_mask_topk_ids_padded_region(routed_cols, num_token_non_padded)
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
recorder_topk_ids = routed_cols
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info
)
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
if recorder_topk_ids is None and not recorder_was_fused:
recorder_topk_ids = (
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
@@ -2598,6 +2732,14 @@ def _post_process_topk_ids(
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
if _is_npu:
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
if recorder_topk_ids is not None and num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_topk_ids = recorder_topk_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
# Shared-expert append/remap can introduce non-zero weights after the
# initial HIP padding mask above. Ensure padded tokens leave this helper
@@ -2750,10 +2892,6 @@ def select_experts(
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
)
elif torch_native and custom_routing_function is None:
assert num_token_non_padded is None, (
"num_token_non_padded is not yet supported in fused_topk_native"
)
assert expert_location_dispatch_info is None
fold_scale_after_routing = True
topk_weights, topk_ids = fused_topk_native(
hidden_states=hidden_states,
@@ -387,6 +387,9 @@ class MuseSpark1xFp4Experts(nn.Module):
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
from sglang.srt.layers.moe.token_dispatcher.standard import (
_map_topk_ids_to_local_experts,
)
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
hidden_states = hidden_states.contiguous()
@@ -398,7 +401,7 @@ class MuseSpark1xFp4Experts(nn.Module):
local_topk_ids = topk_ids.to(torch.int32)
if expert_map is not None:
local_topk_ids = expert_map[local_topk_ids]
local_topk_ids = _map_topk_ids_to_local_experts(local_topk_ids, expert_map)
local_topk_ids = local_topk_ids.contiguous()
topk_weights = topk_scores.to(torch.float32).contiguous()
@@ -430,13 +433,13 @@ class MuseSpark1xFp4Experts(nn.Module):
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache13 = torch.empty(
(num_tokens * topk * max(gemm1_n, size_k),),
cache13 = torch.zeros(
(num_tokens * topk * (gemm1_n + size_k),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k)
cache1 = moe_wna16_marlin_gemm(
hidden_states,
@@ -525,6 +528,7 @@ class MuseSpark1xFp4Experts(nn.Module):
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
hidden_states, topk_scores, topk_ids
)
topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0)
if self._post_expert_norm is not post_expert_norm:
self.set_post_expert_norm(post_expert_norm)
if self._post_expert_norm_weight is not None:
+39 -15
View File
@@ -18,6 +18,9 @@ from sglang.srt.hardware_backend.npu.quantization.moe_methods import (
from sglang.srt.hardware_backend.npu.utils import npu_format_cast
from sglang.srt.layers.linear import LinearBase
from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
moe_align_block_size,
)
from sglang.srt.layers.moe.utils import MoeRunnerBackend, get_moe_runner_backend
from sglang.srt.layers.quantization.base_config import (
FusedMoEMethodBase,
@@ -41,24 +44,22 @@ _is_musa = is_musa()
_is_npu = is_npu()
if _is_cuda:
from sgl_kernel import moe_align_block_size, moe_sum
from sgl_kernel import moe_sum
from sgl_kernel.quantization import (
ggml_dequantize,
ggml_moe_a8,
ggml_moe_a8_vec,
ggml_moe_get_block_size,
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
elif _is_musa:
from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul
from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul
from sgl_kernel.quantization import (
ggml_dequantize,
ggml_moe_a8,
ggml_moe_a8_vec,
ggml_moe_get_block_size,
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
@@ -180,6 +181,16 @@ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
def ggml_moe_get_block_size(qweight_type: int) -> int:
# MOE_X in the CUDA/MUSA GGUF MMQ kernels is 4. The released wheel's
# Tensor-free metadata operator has no catch-all dispatch registration.
return (
(8 if _is_hip else 4)
if qweight_type in MMQ_QUANT_TYPES and qweight_type != WeightType.Q8_1
else 0
)
def dequantize_gguf_weight(
qweight: torch.Tensor, qweight_type: int, dtype: torch.dtype
) -> torch.Tensor:
@@ -239,6 +250,8 @@ def fused_moe_gguf(
return gelu_and_mul(x)
raise ValueError(f"Unsupported activation: {activation}")
inactive = topk_ids < 0
x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0)
out_hidden_states = torch.empty_like(x)
# unless we decent expert reuse we are better off running moe_vec kernel
if (
@@ -254,6 +267,11 @@ def fused_moe_gguf(
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids, BLOCK_SIZE, E
)
# Released MMQ kernels inspect one slack block before their bound check.
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
)
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
out = ggml_moe_a8(
x,
w1,
@@ -265,6 +283,7 @@ def fused_moe_gguf(
top_k,
num_tokens,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = ggml_moe_a8(
out,
@@ -277,8 +296,10 @@ def fused_moe_gguf(
1,
num_tokens * top_k,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
topk_weights.view(num_tokens, top_k, 1)
valid_weights.view(num_tokens, top_k, 1)
)
# TODO(FlamingoPg): maybe we can use moe_sum_reduce here?
moe_sum(out, out_hidden_states)
@@ -287,14 +308,20 @@ def fused_moe_gguf(
E, N, _ = w1.shape
top_k = topk_ids.shape[1]
out = ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
# Older vector kernels form weight addresses before checking -1.
# They have no per-expert capacity buffer; discard their dummy slots.
safe_ids = topk_ids.clamp_min(0)
out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = ggml_moe_a8_vec(
out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
topk_weights.view(num_tokens, top_k, 1)
valid_weights.view(num_tokens, top_k, 1)
)
moe_sum(out, out_hidden_states)
else:
@@ -305,8 +332,10 @@ def fused_moe_gguf(
)
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
inp = x[tok].reshape((1,) + x.shape[1:])
current_hidden_state = None
current_hidden_state = torch.zeros_like(inp)
for ww, ii in zip(w, idx):
if ii < 0:
continue
expert_up = w1[ii]
out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
@@ -316,10 +345,7 @@ def fused_moe_gguf(
current_state = fused_mul_mat_gguf(
out, expert_down, qweight_type2
).mul_(ww)
if current_hidden_state is None:
current_hidden_state = current_state
else:
current_hidden_state.add_(current_state)
current_hidden_state.add_(current_state)
out_hidden_states[tok] = current_hidden_state
return out_hidden_states
@@ -565,8 +591,6 @@ class GGUFMoEMethod(FusedMoEMethodBase):
layer: torch.nn.Module,
dispatch_output: StandardDispatchOutput,
) -> CombineInput:
assert self.fused_experts is None
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
assert self.moe_runner_config.activation == "silu", (
@@ -71,8 +71,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
assert runner_config.activation == "silu", "Only SiLU activation is supported."
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
@@ -119,9 +119,13 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
)
# Stage 1: Gate/Up (Marlin)
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -165,11 +169,6 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: Down (Marlin)
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
if quant_info.expert_map is not None:
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -1,7 +1,7 @@
"""No-sort B200 decode kernels for Inkling routed MoE LoRA.
The contract is BF16 CUDA, M<=32, top-k 6, rank 32, up to four slots, TP-local
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)``.
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)`` or the inactive sentinel ``-1``.
"""
from __future__ import annotations
@@ -47,7 +47,8 @@ def _direct_gate_expand_kernel(
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_r = tl.arange(0, RANK)
@@ -110,7 +111,8 @@ def _direct_down_shrink_kernel(
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_r = tl.arange(0, RANK)
offs_k = tl.arange(0, BLOCK_K)
@@ -121,7 +123,7 @@ def _direct_down_shrink_kernel(
k_mask = k < INTERMEDIATE_SIZE
x = tl.load(
activation_ptr + route * stride_xq + k[None, :] * stride_xk,
mask=(offs_m[:, None] == 0) & k_mask[None, :],
mask=active & (offs_m[:, None] == 0) & k_mask[None, :],
other=0.0,
)
a = tl.load(
@@ -232,8 +232,8 @@ def fused_experts_experimental_sgl_marlin_lora(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
if (
topk_ids.ndim != 2
or topk_weights.shape != topk_ids.shape
@@ -345,7 +345,7 @@ def fused_experts_experimental_sgl_marlin_lora(
!= lora_info.gate_up_lora_b_weights.shape[-1]
)
# Allocate and prewarm on main; side streams launch kernels only.
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
if not direct_decode:
merged_experts_fused_moe_lora_add(
output=gate_up_delta,
@@ -572,9 +572,13 @@ def fused_experts_experimental_sgl_marlin_lora(
)
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -657,8 +661,7 @@ def fused_experts_experimental_sgl_marlin_lora(
local_expert_offset=0,
local_num_experts=E,
zero_intermediate=(
ep_active
and lora_info.experts_shared_outer_loras
lora_info.experts_shared_outer_loras
and not factored_shared_outer
),
)
@@ -679,11 +682,6 @@ def fused_experts_experimental_sgl_marlin_lora(
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: down (marlin).
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
if ep_active:
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -764,7 +762,7 @@ def fused_experts_experimental_sgl_marlin_lora(
routing_cache=routing_cache,
local_expert_offset=0,
local_num_experts=E,
zero_intermediate=(ep_active and lora_info.experts_shared_outer_loras),
zero_intermediate=lora_info.experts_shared_outer_loras,
)
# Never alias hidden_states: the shared sink still reads it. The fused B200
@@ -866,9 +864,7 @@ def fused_experts_experimental_sgl_marlin_lora(
down_intermediate if down_shrink_done is not None else None
),
zero_intermediate=(
ep_active
and lora_info.experts_shared_outer_loras
and down_shrink_done is None
lora_info.experts_shared_outer_loras and down_shrink_done is None
),
)
@@ -835,7 +835,9 @@ class CPUGraphRunner:
spec_algorithm=self.model_runner.spec_algorithm,
spec_info=spec_info,
capture_hidden_mode=self.capture_hidden_mode,
num_token_non_padded=self.num_token_non_padded,
num_token_non_padded=(
self.num_token_non_padded if enable_num_token_non_padded() else None
),
global_forward_mode=self.capture_forward_mode,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
@@ -849,7 +851,8 @@ class CPUGraphRunner:
)
with skip_ctx:
with forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=forward_batch,
):
self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph(
bs,
@@ -991,11 +994,15 @@ class CPUGraphRunner:
with replay_context():
with skip_ctx:
prepared_forward_batch = self.prepare_replay(forward_batch, skip=skip)
output = graphs[prepared_forward_batch.batch_size](
prepared_forward_batch.input_ids,
prepared_forward_batch.positions,
prepared_forward_batch,
)
with forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=prepared_forward_batch,
):
output = graphs[prepared_forward_batch.batch_size](
prepared_forward_batch.input_ids,
prepared_forward_batch.positions,
prepared_forward_batch,
)
if forward_batch.batch_size in graphs:
return output
@@ -1143,14 +1143,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
batch.extend_input_logprob_token_ids.to(device, non_blocking=True)
)
num_tokens = len(batch.input_ids) if batch.input_ids is not None else 0
if enable_num_token_non_padded():
ret.global_num_token_non_padded = torch.tensor(
num_tokens,
dtype=torch.int32,
pin_memory=is_pin_memory_available(device),
).to(device, non_blocking=True)
ret.global_num_token_non_padded_cpu = num_tokens
ret._init_num_token_non_padded(
device=device, is_draft_worker=model_runner.is_draft_worker
)
ret.init_mlp_sync_metadata(
batch, device, is_draft_worker=model_runner.is_draft_worker
@@ -1341,6 +1336,26 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
pin_memory=is_pin_memory_available(batch.device),
).to(batch.device, non_blocking=True)
def _init_num_token_non_padded(self, *, device, is_draft_worker: bool) -> None:
if self.input_ids is not None:
num_tokens = len(self.input_ids)
elif (
is_draft_worker
and self.spec_info is not None
and self.forward_mode.is_decode()
):
assert self.spec_info.num_tokens_per_req > 0
num_tokens = self.batch_size * self.spec_info.num_tokens_per_req
else:
num_tokens = 0
if enable_num_token_non_padded():
self.global_num_token_non_padded = torch.tensor(
num_tokens,
dtype=torch.int32,
pin_memory=is_pin_memory_available(device),
).to(device, non_blocking=True)
self.global_num_token_non_padded_cpu = num_tokens
def set_local_num_token_non_padded(self, *, sharded: bool) -> None:
"""Derive the LOCAL num_token_non_padded from the invariant GLOBAL scalar.
@@ -5,6 +5,8 @@ layer reads at depth via get_forward_context(). The only mandatory field
today is attn_backend; pool refs are derived from attn_backend.*
(every backend caches them at __init__), so a published ForwardContext
is enough to resolve the active pools without a separate global.
The active batch lends its MoE bound getter so late preparation and TBO child
selection stay visible without retaining a parent tensor snapshot.
ModelRunner._forward_raw publishes a fresh ForwardContext for the
duration of each forward; callers that need a per-call override (PDmux
@@ -23,12 +25,15 @@ worker threads ever share a process, migrate to contextvars.ContextVar.
from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Callable, Optional
if TYPE_CHECKING:
import torch
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@dataclass(frozen=True, slots=True)
@@ -38,6 +43,8 @@ class ForwardContext:
write time — use dataclasses.replace for per-call overrides."""
attn_backend: AttentionBackend
# Resolve after batch preparation; captured calls retain the static buffer.
moe_num_token_non_padded: Optional[Callable[[], Optional[torch.Tensor]]] = None
_current: Optional[ForwardContext] = None
@@ -76,7 +83,15 @@ def get_req_to_token_pool() -> ReqToTokenPool:
@contextmanager
def forward_context(ctx: ForwardContext):
def forward_context(
ctx: ForwardContext, *, forward_batch: Optional[ForwardBatch] = None
):
if forward_batch is not None:
ctx = replace(
ctx, moe_num_token_non_padded=forward_batch.moe_num_token_non_padded
)
elif ctx.moe_num_token_non_padded is None and _current is not None:
ctx = replace(ctx, moe_num_token_non_padded=_current.moe_num_token_non_padded)
prev = set_forward_context(ctx)
try:
yield
@@ -113,6 +113,7 @@ from sglang.srt.model_executor.forward_batch_info import (
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.graph_memory_usage import (
@@ -1965,9 +1966,14 @@ class ModelRunner:
if prepare is not None:
prepare(forward_batch)
if has_forward_context():
ctx_mgr = contextlib.nullcontext()
ctx_mgr = forward_context(
get_forward_context(), forward_batch=forward_batch
)
else:
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
ctx_mgr = forward_context(
ForwardContext(attn_backend=self.attn_backend),
forward_batch=forward_batch,
)
with (
get_model().override(shared_experts=self.model.shared_experts_config),
ctx_mgr,
@@ -762,7 +762,9 @@ class BaseRunner(ABC):
torch.get_device_module(mr.device).synchronize()
get_parallel().tp_group.barrier()
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
with forward_context(
ForwardContext(attn_backend=mr.attn_backend), forward_batch=forward_batch
):
with run_ctx or empty_context():
run_once()
@@ -1253,7 +1253,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
# All setup hooks below read get_attn_backend() (TboForwardBatchPreparer,
# DeepEP adapter, …) so they must run inside the same ForwardContext
# that wraps the warmup/capture forward.
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens)
if forward_batch.lora_ids is not None:
@@ -1262,60 +1264,67 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
def run_once():
# Backends may gather live pool mappings into their static
# metadata here; these nodes run before any model KV reads.
attn_backend.init_forward_metadata_in_graph(forward_batch)
self._record_in_graph_metadata_prep_done()
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
None
)
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
num_tokens,
forward_batch.dp_padding_mode.is_max_len(),
forward_batch.global_num_tokens_cpu,
)
set_is_extend_in_batch(False)
kwargs = {}
if (
self.pp_size > 1
and "pp_proxy_tensors" in inspect.signature(forward).parameters
# Debug replay retains this callback beyond the capture scope.
with forward_context(
ForwardContext(attn_backend=attn_backend),
forward_batch=forward_batch,
):
kwargs["pp_proxy_tensors"] = PPProxyTensors(
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
)
if (
(
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
if self.dllm_input_preparation is None:
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
else:
state = self.buffer_registry.get_slot(
"dllm_input_preparation_state"
).buffer[:num_tokens]
kwargs["input_embeds"] = self.dllm_input_preparation(
self.model_runner.model, forward_batch.input_ids, state
)
# Backends may gather live pool mappings into their static
# metadata here; these nodes run before any model KV reads.
attn_backend.init_forward_metadata_in_graph(forward_batch)
self._record_in_graph_metadata_prep_done()
out = forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
**kwargs,
)
for capture_hook in self.model_runner.capture_tail_hooks:
capture_hook(self, out, forward_batch, num_tokens)
return out
forward_batch.dp_local_start_pos = (
forward_batch.dp_local_num_tokens
) = None
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
num_tokens,
forward_batch.dp_padding_mode.is_max_len(),
forward_batch.global_num_tokens_cpu,
)
set_is_extend_in_batch(False)
kwargs = {}
if (
self.pp_size > 1
and "pp_proxy_tensors" in inspect.signature(forward).parameters
):
kwargs["pp_proxy_tensors"] = PPProxyTensors(
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
)
if (
(
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
if self.dllm_input_preparation is None:
kwargs["input_embeds"] = self.buffers.input_embeds[
:num_tokens
]
else:
state = self.buffer_registry.get_slot(
"dllm_input_preparation_state"
).buffer[:num_tokens]
kwargs["input_embeds"] = self.dllm_input_preparation(
self.model_runner.model, forward_batch.input_ids, state
)
out = forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
**kwargs,
)
for capture_hook in self.model_runner.capture_tail_hooks:
capture_hook(self, out, forward_batch, num_tokens)
return out
self.deepep_adapter.capture(is_extend_in_batch=False)
canary_ctx = (
@@ -46,6 +46,7 @@ from sglang.srt.model_executor.forward_batch_info import (
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
get_req_to_token_pool,
get_token_to_kv_pool,
)
@@ -259,7 +260,11 @@ class EagerRunner(BaseRunner):
ctx = device_timer_ctx(model_runner.device_timer, "decode")
with ctx, pdmux_ctx:
with (
ctx,
pdmux_ctx,
forward_context(get_forward_context(), forward_batch=forward_batch),
):
return model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
@@ -341,7 +346,10 @@ class EagerRunner(BaseRunner):
if forward_batch.forward_mode.is_target_verify()
else "extend"
)
with device_timer_ctx(model_runner.device_timer, category):
with (
device_timer_ctx(model_runner.device_timer, category),
forward_context(get_forward_context(), forward_batch=forward_batch),
):
if cp_active and (
not get_parallel().enable_cp_tp_group_sharing
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
@@ -448,7 +456,10 @@ class EagerRunner(BaseRunner):
model_runner.attn_backend.forward_metadata = None
kwargs = model_runner._pp_kwargs(pp_proxy_tensors)
with device_timer_ctx(model_runner.device_timer, "idle"):
with (
device_timer_ctx(model_runner.device_timer, "idle"),
forward_context(get_forward_context(), forward_batch=forward_batch),
):
return model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
@@ -819,7 +819,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
full_graph=self._is_full_backend, raw_num_tokens=raw_num_tokens
),
forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=forward_batch,
),
set_tc_piecewise_forward_context(
forward_batch,
@@ -953,7 +954,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
set_forward_batch(fb),
prefill_graph_scope(full_graph=self._is_full_backend),
forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=fb,
),
set_tc_piecewise_forward_context(
fb,
@@ -1234,7 +1236,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
and stash the returned per-bucket metadata object; otherwise fall
back to the generic eager init that BCG/TC_PIECEWISE use today."""
attn_backend = self.model_runner.attn_backend
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
if not self.use_captured_attn_metadata:
attn_backend.init_forward_metadata(forward_batch)
return
@@ -20,7 +20,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
set_graph_pool_id,
)
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.forward_context import (
forward_context,
get_attn_backend,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
BreakableCUDAGraph,
BreakableCUDAGraphCapture,
@@ -120,7 +125,14 @@ class RowBucketGraphRunner:
# The eager breaks read the step's real row count off this view.
step_batch = copy.copy(forward_batch)
step_batch.global_num_token_non_padded_cpu = num_rows
with context(step_batch):
with (
(
forward_context(get_forward_context(), forward_batch=step_batch)
if has_forward_context()
else nullcontext()
),
context(step_batch),
):
if (rows, graph_key) not in self._graphs:
# Largest first, so smaller buckets reuse the pool memory the
# larger ones took; a bucket below this step's rows cannot be
@@ -168,7 +180,14 @@ class RowBucketGraphRunner:
forward_batch.num_token_non_padded, num_rows
)
capture_batch.num_token_non_padded = num_token_non_padded
with batch_scope(capture_batch, inputs):
with (
(
forward_context(get_forward_context(), forward_batch=capture_batch)
if has_forward_context()
else nullcontext()
),
batch_scope(capture_batch, inputs),
):
graph = self._record(
lambda: body(inputs, capture_batch),
rows,
-5
View File
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -492,10 +491,6 @@ class BailingMoESparseMoeBlock(nn.Module):
hidden_states,
router_logits,
dynamic_expert_bias=dynamic_expert_bias,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -21,7 +21,6 @@ from sglang.srt.configs import KimiLinearConfig
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
@@ -832,10 +831,6 @@ class BailingMoE(nn.Module):
hidden_states,
router_logits,
dynamic_expert_bias=dynamic_expert_bias,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
final_hidden_states = self.experts(hidden_states, topk_output)
+7 -52
View File
@@ -66,7 +66,6 @@ from sglang.srt.distributed import divide
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.amx_utils import PackWeightMethod
@@ -120,10 +119,10 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
DispatchOutput,
)
from sglang.srt.layers.moe.topk import (
BypassedTopKOutput,
TopK,
TopKOutputFormat,
fused_shared_expert_weight,
is_expert_remapping_enabled,
is_routed_scaling_factor_fused_in_topk,
)
from sglang.srt.layers.moe.utils import (
@@ -1038,7 +1037,8 @@ class DeepseekV2MoE(nn.Module):
and not self._enable_a2a_moe
and not self._fuse_shared_experts_inside_sbo
and not getattr(self, "is_hash", False)
and not get_exec().moe.enable_eplb
and not is_expert_remapping_enabled()
and get_exec().moe.ep_dispatch_algorithm not in ("fake", "lp")
)
def forward(
@@ -1067,11 +1067,6 @@ class DeepseekV2MoE(nn.Module):
input_ids_global=input_ids_global,
)
num_token_non_padded = (
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
)
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
if use_vision_topk and _is_hip:
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
@@ -1109,7 +1104,6 @@ class DeepseekV2MoE(nn.Module):
gemm_output_zero_allocator,
input_ids,
input_ids_global=input_ids_global,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
@@ -1121,7 +1115,6 @@ class DeepseekV2MoE(nn.Module):
input_ids,
input_ids_global=input_ids_global,
skip_shared_experts=skip_shared_experts,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
@@ -1137,7 +1130,6 @@ class DeepseekV2MoE(nn.Module):
gemm_output_zero_allocator: BumpAllocator = None,
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
@@ -1167,11 +1159,6 @@ class DeepseekV2MoE(nn.Module):
has_shared_output = (
hidden_states.shape[0] > 0 and self.num_fused_shared_experts == 0
)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb and not self.is_nextn
else None
)
# router_logits: (num_tokens, n_experts)
if _is_hip:
@@ -1185,11 +1172,7 @@ class DeepseekV2MoE(nn.Module):
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
router_logits_partials = None
if use_flashinfer_trtllm_bypass:
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
topk_config=self.topk.topk_config,
)
topk_output = self.topk.forward_cuda(hidden_states, router_logits)
else:
topk_kwargs = (
{"input_ids": input_ids_global}
@@ -1203,14 +1186,11 @@ class DeepseekV2MoE(nn.Module):
self,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
)
else:
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
# Issued after the router so the main chain stays on the main stream at replay.
@@ -1323,7 +1303,6 @@ class DeepseekV2MoE(nn.Module):
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
skip_shared_experts: bool = False,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
@@ -1332,11 +1311,6 @@ class DeepseekV2MoE(nn.Module):
self.shared_experts.gate_up_proj
):
return self.forward_cpu(hidden_states)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb and not self.is_nextn
else None
)
defer_shared = not self.experts.moe_runner_config.inplace
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): shared expert is computed on the LOCAL
# hidden in the decoder layer (before the dp gather) and added after the
@@ -1383,21 +1357,18 @@ class DeepseekV2MoE(nn.Module):
self,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
)
else:
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
else:
pre_quant_input = None
shared_output = None
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
if self._fuse_shared_experts_inside_sbo and not skip_shared_experts:
@@ -1591,19 +1562,11 @@ class DeepseekV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
**topk_kwargs,
)
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
if sbo_overlap_dispatch_flag:
@@ -1979,19 +1942,11 @@ class DeepseekV2MoE(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
**topk_kwargs,
)
else:
state.topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
def op_dispatch_a(self, state):
@@ -28,7 +28,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.forward_context import (
forward_context,
get_attn_backend,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.model_runner_components.layer_setup import (
compute_attention_and_moe_layers,
)
@@ -163,6 +168,15 @@ class DecoderReplayTail:
max_rows=max_rows,
)
@staticmethod
def _can_capture_moe_bound(forward_batch: ForwardBatch) -> bool:
# The row runner replaces only the local count with a graph-owned buffer.
bound = forward_batch.moe_num_token_non_padded()
return bound is None or (
bound is forward_batch.num_token_non_padded
and bound is not forward_batch.global_num_token_non_padded
)
def plan(
self,
tail: Optional[LateLayerTail],
@@ -199,6 +213,7 @@ class DecoderReplayTail:
or get_global_expert_distribution_recorder().recording
or not self.runner.warm
or (tail is not None and tail.cp_metadata is not None)
or not self._can_capture_moe_bound(forward_batch)
):
return False, None
if not dp:
@@ -414,7 +429,14 @@ class DecoderReplayTail:
gathered = input_ids.new_empty((sum(dp_rows), 1))
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
input_ids_global = gathered.squeeze(-1)
with self._break_context(tail_batch):
with (
(
forward_context(get_forward_context(), forward_batch=tail_batch)
if has_forward_context()
else nullcontext()
),
self._break_context(tail_batch),
):
return self._layer_stack(
state,
positions=positions,
@@ -681,6 +703,7 @@ def _tail_layers_in_prefill_graph(
if (
not get_flags().dp.capturing_prefill_graph
and runner.warm
and tail_layers._can_capture_moe_bound(forward_batch)
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
):
residual, pre, aux = tail_layers.run(**kwargs)
@@ -51,7 +51,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
@@ -467,14 +466,10 @@ class Dots3MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(
-5
View File
@@ -26,7 +26,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -228,10 +227,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -36,7 +36,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
@@ -653,10 +652,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -30,7 +30,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
@@ -377,10 +376,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -34,7 +34,6 @@ from triton.language.extra import libdevice
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -918,10 +917,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+10
View File
@@ -97,6 +97,7 @@ from sglang.srt.model_loader.utils import (
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
get_stream,
)
@@ -251,6 +252,15 @@ class LongcatFlashMoE(nn.Module):
self.top_k = config.moe_topk
self.zero_expert_num = config.zero_expert_num
self.zero_expert_type = config.zero_expert_type
if (
self.zero_expert_num > 0
and get_exec().moe.ep_dispatch_algorithm is not None
):
raise ValueError(
"LongCat zero experts do not support expert placement remapping. "
"Disable EPLB, redundant experts, custom expert placement, and "
"--ep-dispatch-algorithm for this model."
)
if config.rounter_params_dtype == "float32":
self.rounter_params_dtype = torch.float32
-14
View File
@@ -29,7 +29,6 @@ from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -559,11 +558,6 @@ class MiMoV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if self._uses_expert_remap
else None
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -595,10 +589,6 @@ class MiMoV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -628,10 +618,6 @@ class MiMoV2MoE(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
@@ -614,10 +613,6 @@ class MiniMaxM2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(device=hidden_states.device)
-5
View File
@@ -36,7 +36,6 @@ from sglang.srt.configs.model_config import (
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -535,10 +534,6 @@ class MiniMaxM3MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+3 -35
View File
@@ -29,7 +29,6 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import ReLU2
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
@@ -349,7 +348,6 @@ class NemotronHMoE(nn.Module):
def _forward_core(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
forward_batch: ForwardBatch | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if (
@@ -364,21 +362,16 @@ class NemotronHMoE(nn.Module):
and is_in_breakable_cuda_graph()
and is_sm120()
):
return self._forward_core_prefill_overlap(
hidden_states, num_token_non_padded
)
return self._forward_core_prefill_overlap(hidden_states)
if _is_cuda and (
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
):
return self._forward_core_shared_routed_overlap(
hidden_states, num_token_non_padded
)
return self._forward_core_normal(hidden_states, num_token_non_padded)
return self._forward_core_shared_routed_overlap(hidden_states)
return self._forward_core_normal(hidden_states)
def _forward_core_normal(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# router_scores: [num_tokens, num_experts]
# bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk.
@@ -392,12 +385,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -407,7 +394,6 @@ class NemotronHMoE(nn.Module):
def _forward_core_shared_routed_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
alt_stream = _get_or_create_alt_stream(self.device_module)
@@ -427,12 +413,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -444,7 +424,6 @@ class NemotronHMoE(nn.Module):
def _forward_core_prefill_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# A multi-stream graph per hybrid segment creates one internal stream
# per executable. Keep each graph linear and launch shared experts on
@@ -460,12 +439,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -509,11 +482,6 @@ class NemotronHMoE(nn.Module):
# MoE runner / topk), so final_hidden_states is already scaled.
final_hidden_states, shared_output = self._forward_core(
hidden_states,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
forward_batch=forward_batch,
)
-19
View File
@@ -38,7 +38,6 @@ from sglang.srt.distributed import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -611,14 +610,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -667,16 +658,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.enable_shared_expert_fusion:
topk_output = self._append_shared_to_topk_output(
+1 -28
View File
@@ -28,7 +28,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.layer_boundary import (
append_stages,
@@ -342,20 +341,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb
else None
)
if hidden_states.shape[0] > 0:
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.gate(hidden_states)
topk_output = self.topk(
hidden_states,
router_logits,
expert_location_dispatch_info=dispatch_info,
)
topk_output = self.topk(hidden_states, router_logits)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
@@ -370,10 +359,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -408,14 +393,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
assert TopKOutputChecker.format_is_standard(topk_output), (
"MegaMoE pre-dispatch consumes raw topk ids/weights; "
@@ -454,10 +431,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -13,7 +13,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -159,10 +158,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -6,7 +6,6 @@ import torch.nn.functional as F
from torch import nn
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -256,10 +255,6 @@ class Step3p5MoEMLP(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+2 -7
View File
@@ -38,7 +38,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
@@ -1000,10 +999,6 @@ class XllmSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
),
)
# Apply router scaling factor after renormalization
if self.router_scaling_factor != 1.0:
@@ -1014,7 +1009,7 @@ class XllmSparseMoeBlock(nn.Module):
topk_output.topk_weights = scaled_weights
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(
hidden_states=hidden_states,
@@ -1051,7 +1046,7 @@ class XllmSparseMoeBlock(nn.Module):
if hidden_states.shape[0] == 0:
shared_output = None
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(hidden_states, topk_output)
else:
@@ -2,6 +2,7 @@ import torch
import torch.nn.functional as F
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import transform_select_experts_inputs
from sglang.srt.layers.moe.topk import (
_RENORMALIZE_SUM_EPSILON,
StandardTopKOutput,
@@ -12,19 +13,10 @@ from sglang.srt.layers.moe.topk import (
)
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
has_per_rank_fused_shared_slots,
)
from sglang.srt.utils import is_cuda
def _scale_fused_shared_weights(weights, num_fused_shared_experts, scaling_factor):
# Standard EP replicates the fused shared expert on every rank and all-reduces,
# so the shared columns carry a 1/ep_size factor.
if num_fused_shared_experts and scaling_factor is not None:
weights[:, -num_fused_shared_experts:] *= scaling_factor
return weights
def vision_topk(
moe,
logits,
@@ -32,25 +24,35 @@ def vision_topk(
num_token_non_padded=None,
expert_location_dispatch_info=None,
):
bound, dispatch_info = moe.topk.routing_inputs()
if num_token_non_padded is None:
num_token_non_padded = bound
if expert_location_dispatch_info is None:
expert_location_dispatch_info = dispatch_info
config = moe.topk.topk_config
logits, correction_bias = transform_select_experts_inputs(
logits, moe.gate.e_score_correction_bias, expert_location_dispatch_info
)
correction_bias_vl = moe.gate.e_score_correction_bias_vl
if (
expert_location_dispatch_info is not None
and expert_location_dispatch_info.ep_dispatch_algorithm == "fake"
):
correction_bias_vl = torch.zeros_like(correction_bias_vl)
num_shared_topk_slots = config.num_shared_topk_slots
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if num_shared_topk_slots and not use_mega_moe:
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
"VL routing does not support per-rank fused shared slots"
)
if is_cuda():
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
from sglang.srt.layers.moe.utils import get_moe_runner_backend
# Same admission as _fused_gate_emits_packed_ids: the shared-expert slots
# rescaled below would rewrite weights after the router.
packed_topk = None
if (
num_shared_topk_slots == 0
and expert_location_dispatch_info is None
and not use_mega_moe
and get_moe_runner_backend().is_flashinfer_mxfp4()
and config.runner_backend is not None
and config.runner_backend.is_flashinfer_mxfp4()
):
packed_topk = torch.empty(
(logits.shape[0], config.top_k), dtype=torch.int32, device=logits.device
@@ -58,11 +60,11 @@ def vision_topk(
weights, indices = moe_fused_gate(
logits,
moe.gate.e_score_correction_bias,
correction_bias,
topk=config.top_k,
scoring_func="sqrtsoftplus",
num_fused_shared_experts=num_shared_topk_slots,
bias_alt=moe.gate.e_score_correction_bias_vl,
bias_alt=correction_bias_vl,
input_ids=input_ids,
bias_alt_token_id=moe.config.image_token_id,
renormalize=config.renormalize and config.top_k > 1,
@@ -87,12 +89,12 @@ def vision_topk(
return StandardTopKOutput(weights, indices, logits)
scores = F.softplus(logits.float()).sqrt()
if input_ids is None:
bias = moe.gate.e_score_correction_bias
bias = correction_bias
else:
bias = torch.where(
(input_ids == moe.config.image_token_id)[:, None],
moe.gate.e_score_correction_bias_vl,
moe.gate.e_score_correction_bias,
correction_bias_vl,
correction_bias,
)
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
topk_routed = config.top_k - num_shared_topk_slots
@@ -136,28 +138,19 @@ def _finish_vision_topk(
):
config = moe.topk.topk_config
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if use_mega_moe:
# Use the same EPLB mapping, recording and home-rank shared slots as
# ordinary top-k before handing physical expert IDs to MegaMoE.
indices, weights, recorder_ids = _post_process_topk_ids(
indices,
weights,
config,
logits,
moe.layer_id,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
padded_rows_masked=padded_rows_masked,
)
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids
)
else:
weights = _scale_fused_shared_weights(
weights,
config.num_shared_topk_slots,
config.fused_shared_experts_scaling_factor,
indices, weights, recorder_ids = _post_process_topk_ids(
indices,
weights,
config,
logits,
moe.layer_id,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
padded_rows_masked=padded_rows_masked,
)
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids
)
# Shared-slot remapping can overwrite padded weights; mask last so idle
# and partially padded DP ranks never dispatch a phantom shared expert.
@@ -511,7 +511,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
return ret
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
with forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
):
self.draft_attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
)
@@ -482,7 +482,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
return ret
with forward_context(
ForwardContext(attn_backend=self.draft_extend_attn_backend)
ForwardContext(attn_backend=self.draft_extend_attn_backend),
forward_batch=forward_batch,
):
self.draft_extend_attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
@@ -800,7 +800,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
forward_context(
ForwardContext(
attn_backend=self.draft_attn_backend.attn_backends[i]
)
),
forward_batch=forward_batch,
),
canary_index_ctx,
):
@@ -903,7 +904,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
forward_context(
ForwardContext(
attn_backend=self.draft_attn_backend.attn_backends[i]
)
),
forward_batch=forward_batch,
),
canary_index_ctx,
):
@@ -357,7 +357,10 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
saved_backend_pool = self.draft_attn_backend.token_to_kv_pool
self.draft_attn_backend.token_to_kv_pool = target_pool
try:
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
with forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
):
self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph(
forward_batch
)
@@ -565,7 +565,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
with (
self._target_kv_pool_view(forward_batch),
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
),
):
seed_output = self.draft_model_runner.forward(forward_batch).logits_output
@@ -608,7 +611,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
with (
self._target_kv_pool_view(forward_batch),
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
),
):
logits_output = self.draft_model_runner.forward(
forward_batch
@@ -451,7 +451,9 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
attn_backend.init_forward_metadata_in_graph(forward_batch)
return self._run_step_body(forward_batch, num_tokens, bs)
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
self.deepep_adapter.capture(is_extend_in_batch=True)
shape_key = self._make_graph_key(bs)
@@ -1026,7 +1028,9 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
forward_batch = r.get_forward_batch(bs)
forward_batch = r._postprocess_forward_batch(forward_batch, bs)
attn_backend = self.draft_extend_attn_backend_list[r.step]
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
)
@@ -1037,7 +1041,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
outs = []
for i, (r, forward_batch, num_tokens, attn_backend) in enumerate(items):
attn_backend.init_forward_metadata_in_graph(forward_batch)
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend),
forward_batch=forward_batch,
):
ret = r._run_step_body(forward_batch, num_tokens, bs)
outs.append(ret)
if i < n - 1:
@@ -814,7 +814,10 @@ class UnoWorkerV2(BaseSpecWorker):
else contextlib.nullcontext()
)
attn_context = (
forward_context(ForwardContext(attn_backend=self._uno_draft_attn_backend))
forward_context(
ForwardContext(attn_backend=self._uno_draft_attn_backend),
forward_batch=forward_batch,
)
if self.tree_mode
else contextlib.nullcontext()
)
@@ -0,0 +1,274 @@
"""CUTLASS FP8 maps and reductions skip inactive expert routes."""
import unittest
from collections import defaultdict
from unittest.mock import patch
import torch
from sglang.kernels.ops.moe.shuffle_weighted_sum import (
_shuffle_weighted_sum_kernel,
shuffle_weighted_sum,
)
from sglang.srt.layers.moe import cutlass_moe
from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=12, stage="base-b-kernel-unit", runner_config="1-gpu-large")
class TestCutlassFp8InactiveRoutes(CustomTestCase):
def test_masked_reduction_ignores_poisoned_slots(self):
for dtype in (torch.float16, torch.bfloat16):
with self.subTest(dtype=dtype):
inputs = torch.full((6, 257), float("nan"), device="cuda", dtype=dtype)
inputs[0].fill_(2)
inputs[1].fill_(4)
mapping = torch.tensor(
[0, 1, -1, -1, 0, -1], device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[[0.25, 0.75], [float("nan"), float("nan")], [0.5, float("nan")]],
device="cuda",
dtype=dtype,
)
output = torch.empty((3, 257), device="cuda", dtype=dtype)
shuffle_weighted_sum(inputs, output, mapping, weights)
expected = torch.tensor([3.5, 0, 1], device="cuda", dtype=dtype)[
:, None
].expand_as(output)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_reduction_reuses_kernels_across_token_counts(self):
"""New lengths reuse the reduction and preserve mixed/empty route results."""
kernel = _shuffle_weighted_sum_kernel
lengths = (1, 3, 7, 8, 15, 16, 17, 31, 32, 33, 64, 65)
for dtype in (torch.float16, torch.bfloat16):
with (
self.subTest(dtype=dtype),
patch.object(
kernel, "device_caches", defaultdict(kernel.create_binder)
),
):
cache = kernel.device_caches[torch.cuda.current_device()][0]
specialization_count = None
for strided_weights in (False, True):
for tokens in lengths:
with self.subTest(
tokens=tokens, strided_weights=strided_weights
):
copies = (tokens + 2) // 3
inputs = torch.full(
(tokens * 2, 257),
float("nan"),
device="cuda",
dtype=dtype,
)
inputs[0].fill_(2)
inputs[1].fill_(4)
mapping = (
torch.tensor(
[[0, 1], [1, -1], [-1, -1]],
device="cuda",
dtype=torch.int32,
)
.repeat(copies, 1)[:tokens]
.flatten()
)
weights = torch.tensor(
[
[0.25, 0.75],
[0.5, float("nan")],
[float("nan"), float("nan")],
],
device="cuda",
dtype=dtype,
).repeat(copies, 1)[:tokens]
if strided_weights:
weights = weights.t().contiguous().t()
output = torch.empty(
(tokens, 257), device="cuda", dtype=dtype
)
shuffle_weighted_sum(inputs, output, mapping, weights)
expected = (
torch.tensor([3.5, 2, 0], device="cuda", dtype=dtype)
.repeat(copies)[:tokens, None]
.expand_as(output)
)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
if specialization_count is None:
specialization_count = len(cache)
self.assertEqual(len(cache), specialization_count)
mapping.fill_(-1)
inputs.fill_(float("nan"))
weights.fill_(float("nan"))
output.fill_(float("nan"))
shuffle_weighted_sum(inputs, output, mapping, weights)
self.assertTrue(bool(torch.isfinite(output).all()))
self.assertEqual(int(torch.count_nonzero(output)), 0)
self.assertEqual(len(cache), specialization_count)
def test_native_fp8_padding_survives_graph_replay(self):
if torch.cuda.get_device_capability()[0] not in (9, 10, 12):
self.skipTest("CUTLASS block FP8 requires SM90, SM100 or SM120")
for mxfp8 in (False, True):
if mxfp8 and torch.cuda.get_device_capability()[0] != 10:
continue
with self.subTest(mxfp8=mxfp8):
self._check_native(mxfp8)
def _check_native(self, mxfp8):
experts, tokens, hidden, intermediate = 2, 16, 256, 256
generator = torch.Generator(device="cuda").manual_seed(945)
inputs = (
torch.randn(
(tokens, hidden),
generator=generator,
device="cuda",
dtype=torch.bfloat16,
)
/ 8
)
ids = torch.tensor(
[[0, 1], [1, -1], [-1, -1], [1, 0]] * 4, device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[
[0.25, 0.75],
[0.6, float("nan")],
[float("nan"), float("nan")],
[0.8, 0.2],
]
* 4,
device="cuda",
)
inputs[(ids < 0).all(dim=-1)] = float("nan")
w1 = (
(
torch.randn(
(experts, 2 * intermediate, hidden),
generator=generator,
device="cuda",
)
/ 16
)
.to(torch.float8_e4m3fn)
.transpose(1, 2)
)
w2 = (
(
torch.randn(
(experts, hidden, intermediate), generator=generator, device="cuda"
)
/ 16
)
.to(torch.float8_e4m3fn)
.transpose(1, 2)
)
if mxfp8:
scale1 = torch.full(
(experts, hidden // 32, 2 * intermediate),
127,
device="cuda",
dtype=torch.uint8,
)
scale2 = torch.full(
(experts, intermediate // 32, hidden),
127,
device="cuda",
dtype=torch.uint8,
)
else:
scale1 = torch.ones(
(experts, 2 * intermediate // 128, hidden // 128), device="cuda"
).transpose(1, 2)
scale2 = torch.ones(
(experts, hidden // 128, intermediate // 128), device="cuda"
).transpose(1, 2)
params = CutlassMoEParams(
CutlassMoEType.BlockscaledFP8,
torch.device("cuda"),
experts,
intermediate,
hidden,
)
workspace = torch.empty(32 * 1024 * 1024, device="cuda", dtype=torch.uint8)
def run(x, route_ids, route_weights):
return cutlass_moe.cutlass_fused_experts_fp8(
x,
w1,
w2,
scale1,
scale2,
route_weights,
route_ids,
params.ab_strides_13,
params.c_strides_13,
params.ab_strides_2,
params.c_strides_2,
workspace,
params.a_ptrs,
params.b_ptrs,
params.out_ptrs,
params.a_scales_ptrs,
params.b_scales_ptrs,
params.expert_offsets,
params.problem_sizes1,
params.problem_sizes2,
use_mxfp8=mxfp8,
enable_es=(mxfp8, mxfp8),
)
reference = run(inputs.nan_to_num(), ids.clamp_min(0), weights.nan_to_num())
prepare = cutlass_moe.prepare_moe_input
def checked_prepare(*args):
prepare(*args)
route_ids, offsets, _, _, a_map, c_map = args[:6]
self.assertTrue(bool(((a_map >= 0) & (a_map < tokens)).all()))
self.assertTrue(bool((c_map[route_ids.flatten() < 0] == -1).all()))
self.assertEqual(int(offsets[-1]), int((route_ids >= 0).sum()))
original_empty = torch.empty
def poisoned_empty(*args, **kwargs):
tensor = original_empty(*args, **kwargs)
if tensor.is_floating_point():
tensor.fill_(float("nan"))
elif tensor.dtype == torch.int32:
tensor.fill_(-999)
return tensor
# The pre-fix map allocation fails checked_prepare before an unsafe gather.
with (
patch.object(torch, "empty", poisoned_empty),
patch.object(cutlass_moe, "prepare_moe_input", checked_prepare),
):
actual = run(inputs, ids, weights)
torch.testing.assert_close(actual, reference, rtol=0.025, atol=0.01)
self.assertTrue(bool(torch.isfinite(actual).all()))
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
result = run(inputs, ids, weights)
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
result = run(inputs, ids, weights)
graph.replay()
torch.testing.assert_close(result, reference, rtol=0.025, atol=0.01)
ids.fill_(-1)
inputs.fill_(float("nan"))
weights.fill_(float("nan"))
graph.replay()
self.assertTrue(bool(torch.isfinite(result).all()))
self.assertEqual(int(torch.count_nonzero(result)), 0)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,185 @@
"""W4A8 routing must pack inactive routes after live experts, including at EP1."""
import unittest
from unittest.mock import patch
import torch
import torch.nn.functional as F
from sglang.srt.layers.moe import cutlass_w4a8_moe as w4a8
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=8, stage="base-b-kernel-unit", runner_config="1-gpu-large")
class TestCutlassW4A8InactiveRoutes(CustomTestCase):
@classmethod
def setUpClass(cls):
if not torch.cuda.is_available():
raise unittest.SkipTest("CUDA is not available")
def test_ep1_and_ep2_reorder_match_dense_experts(self):
"""Real CUDA permutation/count/finalize kernels must ignore padded routes.
Only the external CUTLASS GEMMs are replaced with dense expert GEMMs;
this checks routing, not native INT4 GEMM numerics.
"""
experts, hidden, intermediate = 2, 128, 128
ids = torch.tensor(
[[1, 0], [0, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
)
inputs = (
torch.arange(4 * hidden, device="cuda")
.reshape(4, hidden)
.remainder(7)
.float()
- 3
) / 16
inputs = inputs.to(torch.bfloat16)
eye = torch.eye(hidden, device="cuda")
dense_w13 = [
torch.cat((eye * ((e + 1) / 4), eye * ((e + 2) / 4)))
for e in range(experts)
]
dense_w2 = [eye * ((e + 1) / 2) for e in range(experts)]
def dense_reference(dynamic):
scale1 = (
inputs[(ids >= 0).any(dim=-1)].float().abs().amax() / 448.0
if dynamic
else 1.0
)
activated = {}
for token in range(ids.shape[0]):
for slot in range(ids.shape[1]):
expert = int(ids[token, slot])
if expert < 0:
continue
quantized = (inputs[token].float() / scale1).to(torch.float8_e4m3fn)
gate_up = F.linear(
quantized.float() * scale1, dense_w13[expert]
).to(torch.bfloat16)
gate, up = gate_up.float().chunk(2)
activated[token, slot] = F.silu(gate) * up
scale2 = (
torch.stack(list(activated.values())).abs().amax().clamp_min(1e-10)
/ 448.0
if dynamic
else 1.0
)
result = torch.zeros_like(inputs, dtype=torch.float32)
for (token, slot), value in activated.items():
expert = int(ids[token, slot])
quantized = (value / scale2).to(torch.float8_e4m3fn)
down = F.linear(quantized.float() * scale2, dense_w2[expert]).to(
torch.bfloat16
)
result[token] += down.float() * weights[token, slot]
return result
w13_q = torch.zeros(
(experts, 2 * intermediate, hidden // 2), device="cuda", dtype=torch.int8
)
w2_q = torch.zeros(
(experts, hidden, intermediate // 2), device="cuda", dtype=torch.int8
)
scales = torch.ones((experts, 1), device="cuda")
strides = [
torch.zeros((experts, 3), device="cuda", dtype=torch.int64)
for _ in range(8)
]
offsets = torch.empty(experts + 1, device="cuda", dtype=torch.int32)
sizes1 = torch.empty((experts, 3), device="cuda", dtype=torch.int32)
sizes2 = torch.empty_like(sizes1)
def dense_gemm(
out, x, packed, input_scale, _weight_scale, starts, sizes, *args
):
out.fill_(float("nan"))
matrices = dense_w13 if packed is w13_q else dense_w2
for expert in range(experts):
start, count = int(starts[expert]), int(sizes[expert, 1])
if count:
values = x[start : start + count].float() * input_scale
out[start : start + count].copy_(F.linear(values, matrices[expert]))
def run(value, *, dynamic, route_ids=ids, route_weights=weights):
return w4a8.cutlass_w4a8_moe(
value,
w13_q,
w2_q,
scales,
scales,
route_weights,
route_ids,
*strides,
offsets,
sizes1,
sizes2,
a1_scale=None if dynamic else torch.ones(1, device="cuda"),
a2_scale=None if dynamic else torch.ones(1, device="cuda"),
)
for ep_size in (1, 2):
with (
get_parallel().override(
tp_size=ep_size,
tp_rank=0,
attn_tp_size=ep_size,
attn_tp_rank=0,
moe_ep_size=ep_size,
moe_ep_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
),
patch.object(w4a8, "cutlass_w4a8_moe_mm", side_effect=dense_gemm),
):
for dynamic in (False, True):
reference = dense_reference(dynamic)
for poison in (0.0, 1e6, float("inf"), float("nan")):
with self.subTest(
ep_size=ep_size, dynamic=dynamic, poison=poison
):
value = inputs.clone()
value[2].fill_(poison)
route_weights = weights.masked_fill(ids < 0, float("nan"))
actual = run(
value, dynamic=dynamic, route_weights=route_weights
)
torch.testing.assert_close(
offsets,
torch.tensor(
[0, 3, 5], device="cuda", dtype=torch.int32
),
)
self.assertTrue(torch.isfinite(actual).all().item())
torch.testing.assert_close(
actual, reference.to(actual.dtype), rtol=1e-2, atol=1e-3
)
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
inactive = torch.full_like(ids, -1)
actual = run(
torch.full_like(inputs, float("nan")),
dynamic=True,
route_ids=inactive,
route_weights=torch.full_like(weights, float("nan")),
)
self.assertTrue(torch.isfinite(actual).all().item())
self.assertEqual(torch.count_nonzero(actual).item(), 0)
torch.testing.assert_close(
ids,
torch.tensor(
[[1, 0], [0, -1], [-1, -1], [1, 0]],
device="cuda",
dtype=torch.int32,
),
)
if __name__ == "__main__":
unittest.main()
@@ -10,7 +10,6 @@ from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant
from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute_with_scale
from sglang.srt.layers.moe.moe_runner import humming as humming_runner
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.quantization import humming_utils
from sglang.srt.utils import get_device_sm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -37,6 +36,8 @@ class TestHummingFp8Dispatch(CustomTestCase):
raise unittest.SkipTest("Humming FP8 dispatch requires SM90")
def test_fp8_dispatch_preserves_group_scales(self):
from sglang.srt.layers.quantization import humming_utils
generator = torch.Generator(device="cuda").manual_seed(0)
hidden_states = torch.randn(
5,
@@ -258,5 +259,106 @@ class TestHummingFp8Dispatch(CustomTestCase):
)
class TestHummingInactiveRoutes(CustomTestCase):
@classmethod
def setUpClass(cls):
if not torch.cuda.is_available():
raise unittest.SkipTest("CUDA is not available")
def test_indexed_finalize_skips_poisoned_routes_without_ep(self):
from sglang.kernels.ops.moe.moe_fused_mul_sum import moe_fused_mul_sum
ids = torch.tensor(
[[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
)
# A legal leading storage record makes a negative map lookup observable
# without making the test depend on an out-of-allocation device read.
map_storage = torch.tensor([0, 0, -1], device="cuda", dtype=torch.int32)
for dtype in (torch.bfloat16, torch.float16):
for expert_map in (None, map_storage[1:]):
for poison_weights in (False, True):
with self.subTest(
dtype=dtype,
mapped=expert_map is not None,
poison_weights=poison_weights,
):
valid = ids >= 0
if expert_map is not None:
valid &= expert_map[ids.clamp_min(0)] >= 0
values = (
torch.arange(4 * 2 * 128, device="cuda")
.reshape(4, 2, 128)
.remainder(11)
.to(dtype)
)
values[~valid] = float("nan")
route_weights = weights.clone()
if poison_weights:
route_weights[~valid] = float("nan")
expected = (
(
torch.where(valid.unsqueeze(-1), values.float(), 0)
* torch.where(valid, route_weights, 0).unsqueeze(-1)
)
.sum(dim=1)
.to(dtype)
)
actual = moe_fused_mul_sum(
values,
route_weights,
topk_ids=ids,
expert_map=expert_map,
is_ep=False,
)
self.assertTrue(torch.isfinite(actual).all().item())
torch.testing.assert_close(actual, expected)
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
def test_grouped_permutation_skips_inactive_routes_without_ep(self):
from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute, moe_unpermute
ids = torch.tensor(
[[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
)
inputs = (
torch.arange(4 * 128, device="cuda")
.reshape(4, 128)
.remainder(7)
.to(torch.bfloat16)
)
output = torch.full(
(ids.numel(), 128), float("nan"), device="cuda", dtype=inputs.dtype
)
permuted, src2dst, offsets = moe_permute(
inputs, ids, num_experts=2, is_ep=False, outputs=output
)
self.assertTrue((src2dst.view_as(ids)[ids < 0] < 0).all().item())
torch.testing.assert_close(
offsets, torch.tensor([0, 2, 5], device="cuda", dtype=torch.int32)
)
down = torch.full_like(permuted, float("nan"))
reference = torch.zeros_like(inputs)
for expert in range(2):
start, end = int(offsets[expert]), int(offsets[expert + 1])
down[start:end] = permuted[start:end] * (expert + 1)
for token in range(ids.shape[0]):
for slot in range(ids.shape[1]):
expert = int(ids[token, slot])
if expert >= 0:
reference[token] += (
inputs[token] * (expert + 1) * weights[token, slot]
)
actual = moe_unpermute(down, src2dst, ids, weights)
self.assertTrue(torch.isfinite(actual).all().item())
torch.testing.assert_close(actual, reference, rtol=1e-2, atol=1e-2)
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
if __name__ == "__main__":
unittest.main()
@@ -1,17 +1,24 @@
import itertools
import sys
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from sgl_kernel.scalar_type import scalar_types
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
from sglang.srt.layers import zero_copy_context
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import fused_marlin_moe
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config
from sglang.srt.layers.quantization.marlin_utils_fp4 import (
prepare_moe_nvfp4_layer_for_marlin,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
from sglang.srt.utils.common import is_sm80_supported, is_sm90_supported
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_marlin_utils import (
@@ -20,7 +27,7 @@ from sglang.test.test_marlin_utils import (
marlin_quantize,
)
register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=720, stage="base-b", runner_config="1-gpu-large")
def _has_aot_moe_wna16_marlin_gemm() -> bool:
@@ -896,5 +903,448 @@ def test_mxfp4_tp8_load_before_padding(tp_rank):
)
def _marlin_inactive_weights(experts, output_size, input_size, quant_type):
tensors = _setup_moe_weights(
min(experts, 8), output_size, input_size, quant_type, 128, False, torch.bfloat16
)
result = []
for tensor in tensors:
if tensor is None:
result.append(None)
continue
tensor = tensor.repeat((experts // tensor.shape[0],) + (1,) * (tensor.ndim - 1))
# A leading poisoned record makes negative expert indexing observable
# without reading outside an allocation during the failing regression.
storage = tensor.new_empty((experts + 1, *tensor.shape[1:]))
storage[0].fill_(float("nan") if tensor.is_floating_point() else 0)
storage[1:].copy_(tensor)
result.append(storage[1:])
return result
@pytest.mark.parametrize(
"rows,experts,intermediate",
[(1, 8, 128), (4, 128, 128), (16, 8, 256), (64, 128, 128)],
)
@pytest.mark.parametrize(
"quant_type,num_bits",
[(scalar_types.uint4b8, 4), (scalar_types.uint8b128, 8), (scalar_types.uint4, 4)],
)
def test_fused_marlin_moe_inactive_routes(
rows, experts, intermediate, quant_type, num_bits
):
"""EP1 padded graph rows contribute zero through Marlin and its shared caches."""
_check_marlin_inactive_routes(rows, experts, intermediate, quant_type, num_bits)
def test_stock_marlin_lora_inactive_routes():
if torch.cuda.get_device_capability()[0] < 9:
pytest.skip("Stock Marlin LoRA requires SM90 or newer")
_check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "stock_lora")
def test_factored_marlin_lora_inactive_routes():
if torch.cuda.get_device_capability()[0] < 9:
pytest.skip("Experimental Marlin LoRA requires SM90 or newer")
_check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "factored_lora")
def _check_marlin_inactive_routes(
rows, experts, intermediate, quant_type, num_bits, runner_kind="fused"
):
torch.manual_seed(43)
hidden, topk = 256, 2
with get_context().override_server_args(
model_path="dummy",
device="cuda",
moe_runner_backend="auto",
ep_dispatch_algorithm=None,
enable_eplb=False,
enable_waterfill=False,
):
flags = get_flags().moe
flag_values = {name: getattr(flags, name) for name in flags.__struct_fields__}
flag_values["in_speculative_scope"] = False
with (
flags.override(**flag_values),
get_parallel().override(
tp_size=1,
tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
attn_tp_size=1,
attn_tp_rank=0,
attn_dp_size=1,
attn_cp_size=1,
tp_group=None,
),
):
initialize_moe_config()
x = torch.randn((rows, hidden), device="cuda", dtype=torch.bfloat16)
logits = torch.randn((rows, experts), device="cuda", dtype=torch.float32)
w1_ref, w1, s1, z1, g1, p1 = _marlin_inactive_weights(
experts, 2 * intermediate, hidden, quant_type
)
w2_ref, w2, s2, z2, g2, p2 = _marlin_inactive_weights(
experts, hidden, intermediate, quant_type
)
router = TopK(
top_k=topk,
layer_id=0,
renormalize=True,
output_format=TopKOutputFormat.STANDARD,
allow_routed_experts_capture=False,
)
router.bind_runner_backend(MoeRunnerBackend.MARLIN)
live = torch.tensor(rows, device="cuda", dtype=torch.int32)
batch = ForwardBatch(
forward_mode=ForwardMode.EXTEND,
batch_size=1,
input_ids=torch.arange(rows, device="cuda"),
req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64),
seq_lens=torch.tensor([rows], device="cuda", dtype=torch.int32),
out_cache_loc=torch.arange(rows, device="cuda"),
seq_lens_sum=rows,
num_token_non_padded=live,
)
output = torch.full_like(x, float("nan"))
finite_logits = logits.clone()
hook_weights = []
if runner_kind != "fused":
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.marlin import MarlinMoeQuantInfo
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardDispatchOutput,
)
config = MoeRunnerConfig(
num_experts=experts,
num_local_experts=experts,
hidden_size=hidden,
intermediate_size_per_partition=intermediate,
top_k=topk,
params_dtype=x.dtype,
runner_backend=MoeRunnerBackend.MARLIN,
inplace=False,
)
quant_info = MarlinMoeQuantInfo(
w13_qweight=w1,
w2_qweight=w2,
w13_scales=s1,
w2_scales=s2,
w13_g_idx_sort_indices=p1,
w2_g_idx_sort_indices=p2,
weight_bits=num_bits,
w13_g_idx=g1,
w2_g_idx=g2,
w13_qzeros=z1,
w2_qzeros=z2,
)
if runner_kind == "stock_lora":
from sglang.srt.lora.lora_moe_runner_marlin import (
MarlinLoraRunnerCore,
)
from sglang.srt.lora.lora_moe_runners import LoRAHooks
core = MarlinLoraRunnerCore(config)
def after_down(activation, projected, weights, ids):
hook_weights[:] = [weights, ids]
hooks = LoRAHooks(after_down=after_down)
else:
from sglang.srt.lora.marlin_lora_temp.moe_runner import (
_use_shared_outer_factorization,
fused_experts_experimental_sgl_marlin_lora,
)
# A zero adapter must preserve base output, including NaN-padded
# routing weights consumed by the factored rank-sum reduction.
rank = 8
lora_info = SimpleNamespace(
lora_use_virtual_experts=True,
max_lora_rank=rank,
has_active_lora=True,
experts_shared_outer_loras=True,
gate_up_lora_a_weights=x.new_zeros((1, 1, 2 * rank, hidden)),
gate_up_lora_b_weights=x.new_zeros(
(1, experts, 2 * intermediate, rank)
),
down_lora_a_weights=x.new_zeros(
(1, experts, rank, intermediate)
),
down_lora_b_weights=x.new_zeros((1, 1, hidden, rank)),
token_lora_mapping=torch.zeros(
rows, device="cuda", dtype=torch.int32
),
)
assert _use_shared_outer_factorization(lora_info, rows, topk)
def run():
with forward_context(
ForwardContext(attn_backend=None), forward_batch=batch
):
routing = router(x, logits)
if runner_kind != "fused":
dispatch = StandardDispatchOutput(x, None, routing)
if runner_kind == "stock_lora":
return core.run_from_dispatch(
dispatch, quant_info, config, hooks
).hidden_states
return fused_experts_experimental_sgl_marlin_lora(
dispatch, quant_info, config, lora_info
).hidden_states
return fused_marlin_moe(
x,
w1,
w2,
s1,
s2,
logits,
routing.topk_weights,
routing.topk_ids,
g_idx1=g1,
g_idx2=g2,
sort_indices1=p1,
sort_indices2=p2,
w1_zeros=z1,
w2_zeros=z2,
num_bits=num_bits,
)
weights, ids = torch.topk(torch.softmax(logits, dim=-1), topk)
weights /= weights.sum(dim=-1, keepdim=True)
gate_up = torch.einsum("mh,mtih->mti", x, w1_ref[ids])
gate, up = gate_up.chunk(2, dim=-1)
activated = torch.nn.functional.silu(gate) * up
projected = torch.einsum("mti,mthi->mth", activated, w2_ref[ids])
reference = (projected.float() * weights[:, :, None]).sum(1).to(x.dtype)
with patch.object(zero_copy_context, "get_moe_output", return_value=output):
for _ in range(3):
run()
torch.cuda.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = run()
for count in (max(rows - 1, 0), rows, 0, min(rows, 2)):
logits.copy_(finite_logits)
logits[count:].fill_(float("nan"))
live.fill_(count)
actual.fill_(float("nan"))
graph.replay()
expected = reference.clone()
expected[count:].zero_()
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.02)
assert torch.isfinite(actual).all()
assert torch.count_nonzero(actual[count:]) == 0
if hook_weights:
weights, ids = hook_weights
assert torch.count_nonzero(weights[ids < 0]) == 0
def test_moe_wna16_marlin_inactive_output_ownership():
"""An allocated output zeros inactive rows; caller-owned rows stay untouched."""
torch.manual_seed(57)
m, e, k, n, topk, block_m = 1, 8, 256, 256, 2, 8
_, qweight, scales, zeros, g_idx, permutation = _marlin_inactive_weights(
e, n, k, scalar_types.uint4b8
)
x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16)
ids = torch.full((m, topk), -1, device="cuda", dtype=torch.int32)
routing_weights = torch.ones((m, topk), device="cuda", dtype=torch.float32)
sorted_ids, expert_ids, num_post = moe_align_block_size(ids, block_m, e)
workspace = torch.zeros(
torch.cuda.get_device_properties(x.device).multi_processor_count * 4,
device="cuda",
dtype=torch.int32,
)
caller_output = torch.full((m * topk, n), 3.0, device="cuda", dtype=x.dtype)
original_empty = torch.empty
def poisoned_empty(*args, **kwargs):
tensor = original_empty(*args, **kwargs)
if tensor.is_floating_point():
tensor.fill_(float("nan"))
return tensor
for output in (None, caller_output):
with patch.object(torch, "empty", side_effect=poisoned_empty):
result = _run_single_gemm(
moe_wna16_marlin_gemm,
x,
output,
qweight,
scales,
zeros,
g_idx,
permutation,
workspace,
sorted_ids,
expert_ids,
num_post,
routing_weights,
scalar_types.uint4b8,
block_m,
topk,
m,
n,
k,
False,
True,
True,
)
expected = (
torch.zeros_like(result) if output is None else torch.full_like(result, 3.0)
)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
def test_marlin_lora_direct_inactive_routes():
"""Direct LoRA projections overwrite inactive routes with finite zeros."""
from sglang.srt.lora.marlin_lora_temp.direct_decode import (
direct_decode_down_shrink,
direct_decode_gate_expand,
)
torch.manual_seed(51)
rows, topk, experts, intermediate, rank = 3, 6, 8, 384, 32
ids = torch.tensor(
[[0, 1, 2, 3, 4, 5], [-1, -1, -1, -1, -1, -1], [2, -1, 4, 5, -1, 7]],
device="cuda",
dtype=torch.int32,
)
slots = torch.zeros(rows, device="cuda", dtype=torch.int32)
shared = torch.randn((rows, 2 * rank), device="cuda", dtype=torch.bfloat16)
gate_b = (
torch.randn(
(1, experts, 2 * intermediate, rank), device="cuda", dtype=torch.bfloat16
)
* 0.05
)
down_a = (
torch.randn(
(1, experts, rank, intermediate), device="cuda", dtype=torch.bfloat16
)
* 0.05
)
activation = torch.randn(
(rows * topk, intermediate), device="cuda", dtype=torch.bfloat16
)
activation.view(rows, topk, intermediate)[ids < 0] = float("nan")
gate_output = torch.full(
(rows, topk, 2 * intermediate),
float("nan"),
device="cuda",
dtype=torch.bfloat16,
)
down_output = torch.full(
(rows, topk, rank), float("nan"), device="cuda", dtype=torch.bfloat16
)
direct_decode_gate_expand(shared, gate_b, ids, slots, gate_output)
direct_decode_down_shrink(activation, down_a, ids, slots, down_output)
expected_gate = torch.zeros_like(gate_output)
expected_down = torch.zeros_like(down_output)
for token in range(rows):
for route in range(topk):
expert = int(ids[token, route])
if expert < 0:
continue
for half in range(2):
start = half * intermediate
expected_gate[token, route, start : start + intermediate] = (
gate_b[0, expert, start : start + intermediate].float()
@ shared[token, half * rank : (half + 1) * rank].float()
).to(gate_output.dtype)
expected_down[token, route] = (
down_a[0, expert].float() @ activation[token * topk + route].float()
).to(down_output.dtype)
torch.testing.assert_close(gate_output, expected_gate, rtol=0.03, atol=0.01)
torch.testing.assert_close(down_output, expected_down, rtol=0.03, atol=0.01)
assert torch.count_nonzero(gate_output[ids < 0]) == 0
assert torch.count_nonzero(down_output[ids < 0]) == 0
@pytest.mark.parametrize("fused_norm", [False, True])
def test_muse_marlin_inactive_routes(fused_norm):
"""Muse maps global routes once and masks inactive weights before either norm tail."""
from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts
torch.manual_seed(67)
rows, local_experts, hidden, intermediate, topk = 4, 2, 256, 256, 2
dtype = torch.bfloat16
layer = MuseSpark1xFp4Experts.__new__(MuseSpark1xFp4Experts)
torch.nn.Module.__init__(layer)
layer.quant_config = SimpleNamespace(group_size=16)
layer.moe_runner_config = SimpleNamespace(is_gated=True)
layer.params_dtype = dtype
layer.intermediate_size_per_partition = intermediate
layer.num_local_experts = local_experts
layer.global_num_experts = 4
layer._expert_map = torch.tensor([-1, -1, 0, 1], device="cuda", dtype=torch.int32)
references = []
for prefix, n, k in (
("w13", 2 * intermediate, hidden),
("w2", hidden, intermediate),
):
packed, scales, global_scales, refs = zip(
*[make_nvfp4_weight_and_ref(n, k, dtype) for _ in range(local_experts)]
)
for suffix, tensors in (
("weight", packed),
("weight_scale", scales),
("weight_scale_2", global_scales),
):
setattr(
layer,
f"{prefix}_{suffix}",
torch.nn.Parameter(torch.stack(tensors), requires_grad=False),
)
references.append(torch.stack(refs))
prepare_moe_nvfp4_layer_for_marlin(layer)
x = torch.randn((rows, hidden), device="cuda", dtype=dtype) / 20
ids = torch.tensor(
[[2, 3], [3, -1], [0, 2], [-1, -1]], device="cuda", dtype=torch.int32
)
local_ids = torch.tensor(
[[0, 1], [1, -1], [-1, 0], [-1, -1]], device="cuda", dtype=torch.int32
)
weights = torch.tensor(
[[0.25, 0.75], [0.5, 0.5], [0.25, 0.75], [0.5, 0.5]], device="cuda"
)
weights[local_ids < 0] = float("nan")
norm = torch.nn.RMSNorm(hidden, eps=1e-6, device="cuda", dtype=dtype)
layer._post_expert_norm = norm
layer._post_expert_norm_weight = norm.weight.detach() if fused_norm else None
layer._post_expert_norm_eps = norm.eps
layer._post_expert_norm_weight_first = False
expert_output, mapped = layer._marlin_expert_outputs(x, weights, ids)
torch.testing.assert_close(mapped, local_ids, rtol=0, atol=0)
assert torch.count_nonzero(expert_output[local_ids < 0]) == 0
expected_experts = torch.zeros((rows, topk, hidden), device="cuda", dtype=dtype)
w13_ref, w2_ref = references
for row in range(rows):
for route in range(topk):
expert = int(local_ids[row, route])
if expert < 0:
continue
gate_up = x[row] @ w13_ref[expert].T
gate, up = gate_up.chunk(2)
activated = torch.nn.functional.silu(gate) * up
expected_experts[row, route] = activated @ w2_ref[expert].T
torch.testing.assert_close(expert_output, expected_experts, rtol=0.05, atol=0.25)
safe_weights = weights.masked_fill(local_ids < 0, 0.0)
expected = (
(norm(expected_experts).float() * safe_weights[..., None]).sum(1).to(dtype)
)
actual = layer._forward_marlin(x, weights, ids, norm)
torch.testing.assert_close(actual, expected, rtol=0.05, atol=0.25)
assert torch.isfinite(actual).all()
assert torch.count_nonzero(actual[-1]) == 0
assert torch.isnan(weights[local_ids < 0]).all()
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v", "-s"]))
@@ -59,7 +59,26 @@ class TestSelectExpertsPerRankSharedSlots(CustomTestCase):
"has_per_rank_fused_shared_slots",
side_effect=lambda num_fused: num_fused > 0,
),
get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK),
# Model one rank of TP=EP; routing itself issues no collectives.
get_parallel().override(
tp_size=self.EP_SIZE,
tp_rank=self.EP_RANK,
tp_group=None,
attn_tp_size=self.EP_SIZE,
attn_tp_rank=self.EP_RANK,
attn_tp_group=None,
attn_dp_size=1,
attn_dp_rank=0,
attn_cp_size=1,
attn_cp_rank=0,
moe_ep_size=self.EP_SIZE,
moe_ep_rank=self.EP_RANK,
moe_ep_group=None,
moe_tp_size=1,
moe_tp_rank=0,
moe_dp_size=1,
moe_dp_rank=0,
),
):
out = select_experts(
torch.empty(logits.shape[0], 16, device=logits.device),
@@ -0,0 +1,309 @@
"""Padded routes must contribute zero even when every expert is local."""
import unittest
import torch
import torch.nn.functional as F
import triton.language as tl
from sglang.kernels.ops.moe.fused_moe_triton_kernels import invoke_fused_moe_kernel
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton import (
TritonMoeQuantInfo,
TritonRunnerCore,
pre_permute_standard_to_triton,
)
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small")
class TestTritonMoePadding(CustomTestCase):
ROWS, EXPERTS, TOP_K, HIDDEN, INTERMEDIATE = 4, 4, 2, 128, 128
def setUp(self):
super().setUp()
self.enterContext(
get_context().override_server_args(
model_path="dummy",
device="cuda",
moe_runner_backend="triton",
ep_dispatch_algorithm=None,
enable_eplb=False,
enable_waterfill=False,
)
)
moe_flags = get_flags().moe
scoped_flags = {
name: getattr(moe_flags, name) for name in moe_flags.__struct_fields__
}
scoped_flags["in_speculative_scope"] = False
self.enterContext(moe_flags.override(**scoped_flags))
initialize_moe_config()
self.enterContext(
get_parallel().override(
tp_size=1,
tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
attn_tp_size=1,
attn_tp_rank=0,
attn_dp_size=1,
attn_cp_size=1,
tp_group=None,
)
)
torch.manual_seed(123)
@staticmethod
def _expert_view(shape, dtype, *, quantized=False):
# A normal view with an extra leading record makes selecting an inactive
# expert numerically observable. Only actual experts contain finite data.
if quantized:
storage = torch.randint(
120, 137, (shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype
)
else:
storage = (
torch.randn((shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype)
* 0.05
)
storage[0].fill_(float("nan"))
return storage[1:]
def _fixture(self, dtype, quantized=False):
x = torch.randn((self.ROWS, self.HIDDEN), device="cuda", dtype=dtype) * 0.1
logits = torch.tensor(
[
[9.0, 8.0, 1.0, 0.0],
[0.0, 9.0, 8.0, 1.0],
[1.0, 0.0, 9.0, 8.0],
[8.0, 1.0, 0.0, 9.0],
],
device="cuda",
)
shapes = [
(self.EXPERTS, 2 * self.INTERMEDIATE, self.HIDDEN),
(self.EXPERTS, self.HIDDEN, self.INTERMEDIATE),
]
if quantized:
weights = [
self._expert_view(shape, torch.uint8, quantized=True)
for shape in shapes
]
scales = []
for shape in shapes:
storage = torch.full(
(self.EXPERTS + 1, shape[1], shape[2] // 32),
0.01,
device="cuda",
dtype=dtype,
)
storage[0].fill_(float("nan"))
scales.append(storage[1:])
quant = TritonMoeQuantInfo(
*weights,
use_int8_w8a16=True,
w13_scale=scales[0],
w2_scale=scales[1],
block_shape=[0, 32],
)
dense_weights = [
(weight.to(dtype) - 128) * scale.repeat_interleave(32, dim=-1)
for weight, scale in zip(weights, scales)
]
else:
weights = [self._expert_view(shape, dtype) for shape in shapes]
quant = TritonMoeQuantInfo(*weights)
dense_weights = weights
router = TopK(
top_k=self.TOP_K,
layer_id=0,
output_format=TopKOutputFormat.STANDARD,
allow_routed_experts_capture=False,
)
router.bind_runner_backend(MoeRunnerBackend.TRITON)
batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
batch_size=self.ROWS,
input_ids=torch.arange(self.ROWS, device="cuda"),
req_pool_indices=torch.arange(self.ROWS, device="cuda"),
seq_lens=torch.ones(self.ROWS, device="cuda", dtype=torch.int32),
out_cache_loc=torch.arange(self.ROWS, device="cuda"),
seq_lens_sum=self.ROWS,
num_token_non_padded=torch.tensor(2, device="cuda", dtype=torch.int32),
)
config = MoeRunnerConfig(
num_experts=self.EXPERTS,
num_local_experts=self.EXPERTS,
hidden_size=self.HIDDEN,
intermediate_size_per_partition=self.INTERMEDIATE,
top_k=self.TOP_K,
num_fused_shared_experts=0,
params_dtype=dtype,
runner_backend=MoeRunnerBackend.TRITON,
inplace=False,
)
return x, logits, quant, dense_weights, router, batch, config
@staticmethod
def _prepare(x, routes, quant, config):
state = {}
prepared = pre_permute_standard_to_triton(
StandardDispatchOutput(x, None, routes), quant, config, state
)
return prepared, state
def _run(self, x, routes, quant, config):
prepared, state = self._prepare(x, routes, quant, config)
return TritonRunnerCore(config).run(prepared, quant, state).hidden_states
def _dense_reference(self, x, logits, weights):
probabilities = torch.softmax(logits, dim=-1)
scores, ids = probabilities.topk(self.TOP_K, dim=-1)
scores /= scores.sum(dim=-1, keepdim=True)
result = torch.zeros_like(x)
for row in range(x.shape[0]):
for slot in range(self.TOP_K):
expert = int(ids[row, slot])
gate, up = F.linear(x[row], weights[0][expert]).chunk(2)
activated = (F.silu(gate.float()) * up.float()).to(x.dtype)
result[row] += (
F.linear(activated, weights[1][expert]).float() * scores[row, slot]
).to(x.dtype)
return result
def test_runner_observes_updated_batch_bound(self):
"""An inactive row must stay finite and zero through both expert GEMMs."""
for dtype, quantized in [
(torch.bfloat16, False),
(torch.float16, False),
(torch.bfloat16, True),
]:
with self.subTest(dtype=dtype, quantized=quantized):
x, logits, quant, weights, router, batch, config = self._fixture(
dtype, quantized
)
dense = self._dense_reference(x, logits, weights)
# Reuse the batch and its device scalar, as static graph buffers do.
with forward_context(
ForwardContext(attn_backend=None), forward_batch=batch
):
for live in (2, 3, 0, self.ROWS):
with self.subTest(live=live):
batch.num_token_non_padded.fill_(live)
routes = router.forward_cuda(x, logits)
self.assertTrue(torch.all(routes.topk_ids[live:] == -1))
output = self._run(x, routes, quant, config)
self.assertTrue(torch.isfinite(output).all())
torch.testing.assert_close(
output[live:],
torch.zeros_like(output[live:]),
rtol=0,
atol=0,
)
torch.testing.assert_close(
output[:live], dense[:live], rtol=0.04, atol=2e-4
)
if live:
# A cropped input has no inactive routes at all.
cropped_routes = router.forward_cuda(
x[:live], logits[:live]
)
cropped = self._run(
x[:live], cropped_routes, quant, config
)
torch.testing.assert_close(
output[:live], cropped, rtol=0.01, atol=2e-5
)
def test_inactive_routes_preserve_output_layouts(self):
"""Skipping a route must respect sorted, accumulated and half-width outputs."""
x, logits, quant, _, router, batch, config = self._fixture(torch.bfloat16)
with forward_context(ForwardContext(attn_backend=None), forward_batch=batch):
routes = router.forward_cuda(x, logits)
prepared, state = self._prepare(x, routes, quant, config)
count = int(prepared.num_tokens_post_padded)
block = state["config"]["BLOCK_SIZE_M"]
sorted_ids = prepared.sorted_token_ids[:count].tolist()
expert_ids = prepared.expert_ids[: (count + block - 1) // block].tolist()
for layout in ("sorted", "add", "sum", "swiglu"):
with self.subTest(layout=layout):
width = (
self.INTERMEDIATE if layout == "swiglu" else 2 * self.INTERMEDIATE
)
rows = (
count
if layout == "sorted"
else self.ROWS
if layout == "sum"
else self.ROWS * self.TOP_K
)
output = torch.full((rows, width), 0.375, device="cuda", dtype=x.dtype)
expected = output.clone()
for position, route in enumerate(sorted_ids):
expert = expert_ids[position // block]
if expert == -1 or route >= self.ROWS * self.TOP_K:
continue
value = F.linear(x[route // self.TOP_K], quant.w13_weight[expert])
if layout == "swiglu":
gate, up = value[::2], value[1::2]
value = (F.silu(gate.float()) * up.float()).to(x.dtype)
target = (
position
if layout == "sorted"
else route // self.TOP_K
if layout == "sum"
else route
)
if layout in ("add", "sum"):
expected[target] += value
else:
expected[target] = value
invoke_fused_moe_kernel(
x,
quant.w13_weight,
None,
output,
None,
None,
None,
routes.topk_weights,
routes.topk_ids,
prepared.sorted_token_ids,
prepared.expert_ids,
prepared.num_tokens_post_padded,
False,
self.TOP_K,
state["config"],
compute_type=tl.bfloat16,
use_fp8_w8a8=False,
use_int8_w8a8=False,
use_int8_w8a16=False,
use_int4_w4a16=False,
per_channel_quant=False,
filter_expert=False,
c_sorted=layout == "sorted",
fuse_add_to_output=layout == "add",
add_output_mask=torch.ones(
self.ROWS, device="cuda", dtype=torch.bool
),
fuse_sum_all_reduce=layout == "sum",
router_topk=self.TOP_K,
fuse_swiglu=layout == "swiglu",
)
self.assertTrue(torch.isfinite(output).all())
torch.testing.assert_close(output, expected, rtol=0.02, atol=2e-3)
if __name__ == "__main__":
unittest.main()
@@ -4,6 +4,7 @@ from unittest.mock import patch
import torch
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferDispatcher,
_WorkspaceNamespace,
@@ -126,10 +127,20 @@ class TestFlashinferDispatcher(CustomTestCase):
router_logits = torch.tensor(
[[0.1, 3.0, 0.2, 2.0], [5.0, 0.0, 4.0, 0.1], [0.0, 0.1, 0.2, 9.0]]
)
logical_to_physical = torch.tensor([2, 0, 3, 1])
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=TopKConfig(top_k=2, torch_native=True),
expert_location_dispatch_info=ExpertLocationDispatchInfo(
ep_dispatch_algorithm="static",
partial_logical_to_rank_dispatch_physical_map=logical_to_physical,
partial_logical_to_all_physical_map=logical_to_physical[:, None],
partial_logical_to_all_physical_map_num_valid=torch.ones(
4, dtype=torch.int64
),
num_physical_experts=4,
),
)
_dispatch_without_dp(
@@ -142,7 +153,7 @@ class TestFlashinferDispatcher(CustomTestCase):
expert_ids = moe_a2a.token_selected_experts
self.assertEqual(expert_ids.dtype, torch.int32)
self.assertEqual(
expert_ids.sort(dim=-1).values.tolist(), [[1, 3], [0, 2], [2, 3]]
expert_ids.sort(dim=-1).values.tolist(), [[0, 1], [2, 3], [1, 3]]
)
topk_weights = moe_a2a.payloads[moe_a2a.expert_id_payload_index + 1]
self.assertEqual(topk_weights.shape, (3, 2))
@@ -531,13 +531,13 @@ class TestFusedSharedExpertScaling(CustomTestCase):
def test_glm53_idle_rank_preserves_routed_topk_contract(self):
"""An idle rank must return the pre-append top-8 shape; returning top-9
here would make the AITER postprocessor append shared ID 288 twice."""
topk = TopK.__new__(TopK)
torch.nn.Module.__init__(topk)
topk.topk_config = TopKConfig(
top_k=9, shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
)
topk.enable_waterfill = False
topk.waterfill_balancer = None
with (
get_context().override_server_args(enable_waterfill=False),
get_model().override(
shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
),
):
topk = TopK(top_k=8, layer_id=0)
with (
patch.object(
@@ -8,6 +8,7 @@ collectives and graph replay are out of scope.
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
@@ -61,11 +62,6 @@ def runtime(monkeypatch):
monkeypatch.setattr(mega, "get_dp_global_num_tokens", lambda: state.global_tokens)
monkeypatch.setattr(mega, "_ensure_mori_shmem", lambda: None)
monkeypatch.setattr(mega, "_ep_rank_world", lambda: (state.rank, 8))
monkeypatch.setattr(
mega,
"ExpertLocationDispatchInfo",
SimpleNamespace(init_new=lambda layer_id: None),
)
utils = ModuleType("sglang.srt.models.deepseek_common.utils")
utils._use_aiter = True
monkeypatch.setitem(sys.modules, utils.__name__, utils)
@@ -100,6 +96,13 @@ def runtime(monkeypatch):
state.extend = extend
state.capture = capture
state.global_tokens = global_tokens
topk = Mock(
return_value=SimpleNamespace(
topk_ids=torch.zeros(local_rows, TOPK, dtype=torch.int64),
topk_weights=torch.ones(local_rows, TOPK),
)
)
topk.routing_inputs.return_value = (None, None)
moe = SimpleNamespace(
config=SimpleNamespace(
hidden_size=HIDDEN,
@@ -110,10 +113,7 @@ def runtime(monkeypatch):
num_fused_shared_experts=0,
experts=state.layer if layer is None else layer,
gate=lambda h, forward_batch=None: torch.zeros(h.shape[0], EXPERTS),
topk=lambda h, logits, **kwargs: SimpleNamespace(
topk_ids=torch.zeros(h.shape[0], TOPK, dtype=torch.int64),
topk_weights=torch.ones(h.shape[0], TOPK),
),
topk=topk,
is_hash=False,
layer_id=0,
routed_scaling_factor=1.0,
@@ -17,10 +17,17 @@ maybe_stub_sgl_kernel()
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe
from sglang.srt.layers.moe import topk as topk_module
from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
from sglang.srt.layers.moe.topk import TopKConfig
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import draft_model_build_scope
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
from sglang.srt.runtime_context import (
get_context,
get_exec,
get_flags,
get_model,
get_parallel,
)
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -257,11 +264,6 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
),
# Toy shapes.
patch.object(mega_moe, "check_mega_moe_shapes"),
patch.object(
mega_moe.ExpertLocationDispatchInfo,
"init_new",
return_value=object(),
),
get_parallel().override(
moe_ep_group=SimpleNamespace(device_group=object())
),
@@ -356,6 +358,14 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
expert boundary exposes accidentally routing image rows as text, and
padded rows must contribute nothing.
"""
config = get_context().override_server_args(
model_path="dummy",
enable_waterfill=False,
ep_dispatch_algorithm=None,
enable_eplb=False,
)
config.install()
self.addCleanup(config.restore)
for fused_shared in (0, 1):
for scale_in_topk in (False, True):
with self.subTest(
@@ -383,12 +393,17 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
with (
get_flags().moe.override(a2a_backend=MoeA2ABackend.MEGAMOE),
get_parallel().override(moe_ep_size=2, moe_ep_rank=1),
patch.object(
mega_moe.ExpertLocationDispatchInfo,
"init_new",
return_value=None,
get_parallel().override(
tp_size=2,
tp_rank=1,
attn_tp_size=2,
attn_tp_rank=1,
moe_ep_size=2,
moe_ep_rank=1,
moe_tp_size=1,
moe_tp_rank=0,
),
forward_context(ForwardContext(None), forward_batch=batch),
patch.object(mega_moe, "run_mega_routed_experts", run_experts),
# The fused padded-row fill requires Triton/GPU.
patch(
@@ -437,42 +452,46 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
@staticmethod
def _v41_moe(fused_shared, scale_in_topk):
return SimpleNamespace(
config=SimpleNamespace(
hidden_size=4,
num_experts_per_tok=2,
moe_intermediate_size=8,
image_token_id=99,
with (
get_context().override_server_args(),
get_model().override(
shared_experts=SharedExpertsConfig(
num_fused_shared_experts=fused_shared
)
),
experts=SimpleNamespace(
should_fuse_routed_scaling_factor_in_topk=scale_in_topk,
moe_runner_config=SimpleNamespace(swiglu_limit=None),
uses_expert_location=False,
),
gate=MagicMock(
return_value=torch.zeros((3, 4)),
e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]),
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]),
),
topk=SimpleNamespace(
topk_config=TopKConfig(
top_k=2 + fused_shared,
shared_experts=SharedExpertsConfig(
num_fused_shared_experts=fused_shared
),
):
return SimpleNamespace(
config=SimpleNamespace(
hidden_size=4,
num_experts_per_tok=2,
moe_intermediate_size=8,
image_token_id=99,
),
experts=SimpleNamespace(
should_fuse_routed_scaling_factor_in_topk=scale_in_topk,
moe_runner_config=SimpleNamespace(swiglu_limit=None),
),
gate=MagicMock(
return_value=torch.zeros((3, 4)),
e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]),
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]),
),
topk=TopK(
top_k=2,
layer_id=0,
scoring_func="sqrtsoftplus",
renormalize=True,
routed_scaling_factor=2.0,
apply_routed_scaling_factor_on_output=scale_in_topk,
)
),
is_hash=False,
num_fused_shared_experts=fused_shared,
layer_id=0,
routed_scaling_factor=2.0,
routed_scaling_factor_fused_in_topk=scale_in_topk,
mega_shared_l1_weights=None,
mega_shared_l2_weights=None,
)
),
is_hash=False,
num_fused_shared_experts=fused_shared,
layer_id=0,
routed_scaling_factor=2.0,
routed_scaling_factor_fused_in_topk=scale_in_topk,
mega_shared_l1_weights=None,
mega_shared_l2_weights=None,
)
def test_shape_check_uses_pinned_fp8_fp4_alignment(self):
# Packed SF groups need 128 elements, not a 16-byte token SF row.
@@ -0,0 +1,724 @@
"""Routing follows the owning layer and active batch across target/draft and TBO."""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.batch_overlap.operations import (
ExecutionOperation,
_StageExecutor,
)
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.environ import envs
from sglang.srt.eplb import expert_location_dispatch
from sglang.srt.eplb.expert_location import ExpertLocationMetadata
from sglang.srt.hardware_backend.npu.moe.topk import fused_topk_npu
from sglang.srt.layers.dp_attention import DpPaddingMode
from sglang.srt.layers.moe import hash_topk as hash_topk_module
from sglang.srt.layers.moe import topk as topk_module
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.hash_topk import HashTopK
from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
from sglang.srt.layers.moe.topk import (
StandardTopKOutputPacked,
TopK,
TopKOutputFormat,
fused_topk_torch_native,
)
from sglang.srt.layers.moe.utils import (
MoeRunnerBackend,
draft_model_build_scope,
initialize_moe_config,
)
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
has_forward_context,
)
from sglang.srt.multimodal.dsv41.vl_routing import vision_topk
from sglang.srt.runtime_context import (
get_context,
get_exec,
get_model,
get_parallel,
get_resources,
)
from sglang.srt.speculative.eagle_info import EagleDraftInput
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
def _batch(real):
empty = torch.empty(0, dtype=torch.int32)
batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
batch_size=4,
input_ids=torch.arange(4),
req_pool_indices=empty,
seq_lens=empty,
out_cache_loc=empty,
seq_lens_sum=0,
)
batch.num_token_non_padded = torch.tensor(real, dtype=torch.int32)
batch.global_num_token_non_padded = batch.num_token_non_padded
batch.dp_padding_mode = DpPaddingMode.SUM_LEN
batch.global_dp_buffer_len = batch.tbo_padded_len = 4
return batch
def _placement(num_layers=1):
logical_to_physical = torch.tensor([[2, 0, 3, 1]], dtype=torch.int64).repeat(
num_layers, 1
)
physical_to_logical = logical_to_physical.argsort(dim=-1)
all_physical = torch.full((num_layers, 4, 4), -1, dtype=torch.int64)
all_physical[:, :, 0] = logical_to_physical
return ExpertLocationMetadata(
physical_to_logical_map=physical_to_logical,
physical_to_logical_map_cpu=physical_to_logical.clone(),
logical_to_all_physical_map=all_physical,
logical_to_all_physical_map_cpu=all_physical.clone(),
logical_to_all_physical_map_num_valid=torch.ones(
(num_layers, 4), dtype=torch.int64
),
ep_size=1,
logical_to_rank_dispatch_physical_map=logical_to_physical,
)
class TestTopKRoutingInputs(CustomTestCase):
def setUp(self):
super().setUp()
config = get_context().override_server_args(
ep_dispatch_algorithm="static", enable_eplb=True
)
config.install()
self.addCleanup(config.restore)
initialize_moe_config()
topology = get_parallel().override(
attn_dp_size=1, attn_cp_size=1, attn_tp_size=1, attn_tp_rank=0
)
topology.__enter__()
self.addCleanup(topology.__exit__, None, None, None)
for name in ("_is_cuda", "_is_hip", "_use_aiter"):
scope = patch.object(topk_module, name, False)
scope.start()
self.addCleanup(scope.stop)
self.metadata = _placement()
get_resources().expert_location_metadata = self.metadata
self.addCleanup(setattr, get_resources(), "expert_location_metadata", None)
self.hidden = torch.ones((4, 2))
self.logits = torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(4, 1)
def _router(self, **kwargs):
shared_slots = kwargs.pop("num_fused_shared_experts", 0)
layer_id = kwargs.pop("layer_id", 0)
with get_model().override(
shared_experts=SharedExpertsConfig(num_fused_shared_experts=shared_slots)
):
return TopK(
top_k=2 - shared_slots,
layer_id=layer_id,
custom_routing_function=fused_topk_torch_native,
**kwargs,
)
def test_target_and_draft_remap_only_the_target_and_keep_mask(self):
"""Draft routes must not use the target placement or lose every live row."""
target = self._router()
with draft_model_build_scope():
draft = self._router()
batch = _batch(2)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=batch
):
target_ids = target.forward_native(self.hidden, self.logits).topk_ids
draft_ids = draft.forward_native(self.hidden, self.logits).topk_ids
self.assertEqual(target_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]])
self.assertEqual(draft_ids.tolist(), [[0, 1], [0, 1], [-1, -1], [-1, -1]])
self.assertFalse(draft.topk_config.allow_routed_experts_capture)
self.assertTrue(target.topk_config.allow_routed_experts_capture)
def test_late_preparation_and_in_place_rebalance(self):
"""Forward preparation and rebalance must be observed after scope entry."""
router = self._router()
batch = _batch(1)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=batch
):
before = router.forward_native(self.hidden, self.logits).topk_ids
batch.num_token_non_padded = torch.tensor(3)
self.metadata.logical_to_rank_dispatch_physical_map.copy_(
torch.tensor([[1, 3, 0, 2]])
)
after = router.forward_native(self.hidden, self.logits).topk_ids
self.assertEqual(before[0].tolist(), [2, 0])
self.assertEqual(before[1].tolist(), [-1, -1])
self.assertEqual(after.tolist(), [[1, 3], [1, 3], [1, 3], [-1, -1]])
def test_nested_scope_restores_and_retains_replay_buffer_identity(self):
router = self._router()
parent, child = _batch(3), _batch(1)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=parent
):
original = get_forward_context()
bound, _ = router.routing_inputs()
self.assertIs(bound, parent.num_token_non_padded)
with self.assertRaisesRegex(RuntimeError, "child failed"):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=child
):
self.assertIs(
router.routing_inputs()[0], child.num_token_non_padded
)
raise RuntimeError("child failed")
self.assertIs(get_forward_context(), original)
parent.num_token_non_padded.fill_(2)
self.assertIs(router.routing_inputs()[0], bound)
self.assertEqual(
router.forward_native(self.hidden, self.logits).topk_ids[-2:].tolist(),
[[-1, -1], [-1, -1]],
)
self.assertFalse(has_forward_context())
def test_tbo_children_bind_bounds_without_tbo_attention_backend(self):
router = self._router()
seen = []
def read_bound(state, forward_batch):
seen.append(int(router.routing_inputs()[0]))
return {"forward_batch": forward_batch}
stages = [[ExecutionOperation("route", read_bound)]] * 2
parent, child_a, child_b = _batch(4), _batch(1), _batch(3)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=parent
):
a = _StageExecutor("a", stages, {"forward_batch": child_a})
b = _StageExecutor("b", stages, {"forward_batch": child_b})
a.next()
b.next()
a.next()
b.next()
self.assertEqual(int(router.routing_inputs()[0]), 4)
self.assertEqual(seen, [1, 3, 1, 3])
def test_hash_router_uses_same_placement_and_bound(self):
router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0)
with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
):
output = router(self.hidden, self.logits, torch.arange(4))
self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [-1, -1], [-1, -1]])
def test_vision_router_maps_routed_columns_and_masks_shared_rows_last(self):
router = self._router(
num_fused_shared_experts=1,
routed_scaling_factor=2.0,
fused_shared_experts_scaling_factor=0.25,
)
moe = SimpleNamespace(
topk=router,
layer_id=0,
config=SimpleNamespace(image_token_id=99),
gate=SimpleNamespace(
e_score_correction_bias=torch.zeros(4),
e_score_correction_bias_vl=torch.zeros(4),
),
)
with patch(
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False
):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
):
output = vision_topk(moe, self.logits, torch.tensor([1, 99, 1, 99]))
self.assertEqual(output.topk_ids.tolist(), [[2, 4], [2, 4], [-1, -1], [-1, -1]])
torch.testing.assert_close(output.topk_weights[:2, -1], torch.full((2,), 0.125))
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
def test_vision_packed_output_follows_built_runner(self):
"""A requested MXFP4 fallback must not emit an unused packed-ID buffer."""
with get_context().override_server_args(moe_runner_backend="flashinfer_mxfp4"):
initialize_moe_config()
for backend in (
None,
MoeRunnerBackend.TRITON,
MoeRunnerBackend.FLASHINFER_MXFP4,
):
with self.subTest(backend=backend):
router = self._router(is_draft=True)
router.bind_runner_backend(backend)
moe = SimpleNamespace(
topk=router,
layer_id=0,
config=SimpleNamespace(image_token_id=99),
gate=SimpleNamespace(
e_score_correction_bias=torch.zeros(4),
e_score_correction_bias_vl=torch.zeros(4),
),
)
with (
patch(
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda",
return_value=True,
),
patch(
"sglang.kernels.ops.moe.moe_fused_gate.moe_fused_gate",
return_value=(
torch.full((4, 2), 0.5),
torch.tensor([[0, 1]], dtype=torch.int32).repeat(4, 1),
),
) as gate,
):
output = vision_topk(moe, self.logits, input_ids=None)
packed = gate.call_args.kwargs["packed_out"]
expected_packed = backend == MoeRunnerBackend.FLASHINFER_MXFP4
self.assertEqual(packed is not None, expected_packed)
self.assertEqual(
isinstance(output, StandardTopKOutputPacked), expected_packed
)
def test_vision_fake_algorithm_neutralizes_both_biases(self):
with get_exec().moe.override(ep_dispatch_algorithm="fake"):
router = self._router()
moe = SimpleNamespace(
topk=router,
layer_id=0,
config=SimpleNamespace(image_token_id=99),
gate=SimpleNamespace(
e_score_correction_bias=torch.tensor([0.0, 0.0, 1000.0, 0.0]),
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 0.0, 1000.0]),
),
)
with patch(
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False
):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(4)
):
logits = self.logits.clone()
output = vision_topk(moe, logits, torch.tensor([1, 99, 1, 99]))
logical = logits.topk(2, dim=-1).indices
expected = self.metadata.logical_to_all_physical_map[0, logical, 0]
self.assertTrue(torch.equal(output.topk_ids.long(), expected))
def test_draft_without_input_ids_retains_prepared_live_width(self):
"""Deferred draft IDs must not turn a live draft's padding bound into zero."""
batch = _batch(0)
batch.input_ids = None
batch.spec_info = EagleDraftInput(num_tokens_per_req=2)
with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4):
batch._init_num_token_non_padded(device="cpu", is_draft_worker=True)
self.assertEqual(batch.global_num_token_non_padded_cpu, 8)
self.assertEqual(int(batch.global_num_token_non_padded), 8)
batch.forward_mode = ForwardMode.IDLE
with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4):
batch._init_num_token_non_padded(device="cpu", is_draft_worker=True)
self.assertEqual(int(batch.global_num_token_non_padded), 0)
def test_npu_routes_keep_valid_ids_and_zero_only_padded_contributions(self):
"""NPU padding must stay inert without inventing invalid or duplicate routes."""
router = self._router(num_fused_shared_experts=1)
router.topk_config.scoring_func = "sqrtsoftplus"
logical = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32)
weights = torch.tensor([[0.75, 0.25]]).repeat(4, 1)
class Capture:
def capture(self, *, layer_id, topk_indices):
self.ids = topk_indices.clone()
capture = Capture()
get_resources().experts_capturer = capture
self.addCleanup(setattr, get_resources(), "experts_capturer", None)
_, info = router.routing_inputs()
with patch.object(
torch.ops.custom,
"npu_moe_gating_top_k",
create=True,
return_value=(weights, logical, None),
):
output = fused_topk_npu(
hidden_states=self.hidden,
router_logits=self.logits,
topk_config=router.topk_config,
num_token_non_padded=torch.tensor(2),
expert_location_dispatch_info=info,
layer_id=0,
)
self.assertEqual(output.topk_ids.tolist(), [[2, 4], [0, 4], [3, 4], [1, 4]])
torch.testing.assert_close(
output.topk_weights[:2], torch.tensor([[0.75, 0.25]]).repeat(2, 1)
)
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
self.assertEqual(capture.ids.tolist(), [[0], [1], [2], [3]])
def test_in_kernel_formats_do_not_silently_ignore_expert_placement(self):
router = self._router(output_format=TopKOutputFormat.TRITON_KERNEL)
with self.assertRaisesRegex(
NotImplementedError, "placement remapping requires"
):
router.forward_cuda(self.hidden, self.logits)
router = self._router(output_format=TopKOutputFormat.BYPASSED)
output = router.forward_cuda(self.hidden, self.logits)
with self.assertRaisesRegex(
NotImplementedError, "placement remapping requires"
):
_ = output.router_logits
def test_bypassed_materialization_preserves_placement_and_padding(self):
"""A dispatcher can remap lazy routes before sending physical expert IDs."""
router = self._router(output_format=TopKOutputFormat.BYPASSED)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
):
output = router.forward_cuda(self.hidden, self.logits)
materialized = output.to_standard(layer_id=0)
self.assertEqual(
materialized.topk_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]]
)
self.assertIs(materialized.router_logits, self.logits)
layer = SimpleNamespace(
_use_ascend_fuseep=False,
forward_impl=lambda hidden, routes, **kwargs: routes.to_standard(
layer_id=0
),
)
with patch(
"sglang.srt.layers.moe.fused_moe_triton.layer.is_in_tc_piecewise_cuda_graph",
return_value=True,
):
piecewise = FusedMoE.forward(layer, self.hidden, output)
torch.testing.assert_close(piecewise.topk_ids, materialized.topk_ids)
def test_in_range_draft_loads_experts_in_its_routing_order(self):
"""Draft expert weights stay in logical order despite a permuted target map."""
outputs = []
for is_draft in (False, True):
method = UnquantizedFusedMoEMethod()
scope = draft_model_build_scope() if is_draft else get_parallel().override()
with (
scope,
patch.object(method, "create_weights"),
get_parallel().override(
tp_size=1,
moe_ep_size=1,
moe_ep_rank=0,
moe_tp_size=1,
moe_tp_rank=0,
),
):
layer = FusedMoE(
num_experts=4,
hidden_size=2,
intermediate_size=2,
layer_id=0,
top_k=2,
quant_method=method,
)
weights = torch.nn.Parameter(torch.zeros((4, 2, 2)), requires_grad=False)
for expert_id in range(4):
layer.weight_loader(
param=weights,
loaded_weight=torch.eye(2) * (expert_id + 1),
weight_name="experts.w2_weight",
shard_id="w2",
expert_id=expert_id,
)
self.assertEqual(
weights[:, 0, 0].tolist(), [1, 2, 3, 4] if is_draft else [2, 4, 1, 3]
)
router = self._router(is_draft=is_draft)
topk = router.forward_native(self.hidden, self.logits)
expert_outputs = torch.einsum(
"tkij,tj->tki", weights[topk.topk_ids], self.hidden
)
outputs.append((expert_outputs * topk.topk_weights[..., None]).sum(dim=1))
torch.testing.assert_close(outputs[0], outputs[1])
expected_scale = torch.softmax(self.logits[:, :2], dim=-1) @ torch.tensor(
[1.0, 2.0]
)
torch.testing.assert_close(outputs[1], self.hidden * expected_scale[:, None])
def test_identity_placement_keeps_in_kernel_routing_available(self):
identity = torch.arange(4).view(1, 4)
self.metadata.logical_to_rank_dispatch_physical_map.copy_(identity)
self.metadata.logical_to_all_physical_map[:, :, 0].copy_(identity)
self.metadata.physical_to_logical_map.copy_(identity)
with get_exec().moe.override(
enable_eplb=False,
init_expert_location="trivial",
ep_num_redundant_experts=0,
ep_dispatch_algorithm="dynamic",
):
for output_format in (
TopKOutputFormat.BYPASSED,
TopKOutputFormat.TRITON_KERNEL,
):
with self.subTest(output_format=output_format):
router = self._router(output_format=output_format)
with patch.object(
topk_module, "routing", return_value=(None, None, None, None, 2)
):
output = router.forward_cuda(self.hidden, self.logits)
self.assertEqual(output.format, output_format)
if output_format == TopKOutputFormat.BYPASSED:
self.assertIs(output.router_logits, self.logits)
for algorithm in ("fake", "lp"):
with (
self.subTest(algorithm=algorithm),
get_exec().moe.override(ep_dispatch_algorithm=algorithm),
):
router = self._router(output_format=TopKOutputFormat.BYPASSED)
output = router.forward_cuda(self.hidden, self.logits)
with self.assertRaisesRegex(
NotImplementedError, "placement remapping requires"
):
_ = output.router_logits
def test_hash_npu_keeps_valid_ids_and_zero_padded_weights(self):
router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0)
with (
patch.object(hash_topk_module, "_is_npu", True),
envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False),
):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
):
output = router(self.hidden, self.logits, torch.arange(4))
self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [3, 1], [1, 2]])
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
def test_hash_global_shared_column_stays_outside_expert_map(self):
with get_model().override(
shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
):
router = HashTopK(
2,
4,
vocab_size=4,
scoring_func="sqrtsoftplus",
routed_scaling_factor=2.0,
layer_id=0,
)
with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False):
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
):
output = router(self.hidden, self.logits, torch.arange(4))
self.assertEqual(
output.topk_ids.tolist(), [[2, 0, 4], [0, 3, 4], [-1, -1, -1], [-1, -1, -1]]
)
def test_deepseek_opaque_bypass_requires_identity_placement_and_native_policy(self):
from sglang.srt.models import deepseek_v2
moe = SimpleNamespace(
alt_stream=object(),
num_fused_shared_experts=0,
shared_experts=object(),
experts=SimpleNamespace(use_flashinfer_trtllm_moe=True),
_enable_a2a_moe=False,
_fuse_shared_experts_inside_sbo=False,
is_hash=False,
topk=self._router(output_format=TopKOutputFormat.BYPASSED),
)
with (
patch.object(deepseek_v2, "_enable_pcg_dsv2_dual_stream", True),
patch.object(
deepseek_v2, "is_in_tc_piecewise_cuda_graph", return_value=True
),
patch.object(
deepseek_v2,
"get_moe_runner_backend",
return_value=MoeRunnerBackend.FLASHINFER_TRTLLM,
),
get_exec().moe.override(
enable_eplb=False,
init_expert_location="trivial",
ep_num_redundant_experts=0,
ep_dispatch_algorithm=None,
),
):
self.assertTrue(
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
)
with get_exec().moe.override(init_expert_location="reverse"):
self.assertFalse(
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
)
moe.topk.topk_config.output_format = TopKOutputFormat.STANDARD
self.assertFalse(
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
)
def test_lp_keeps_global_shared_columns_outside_routed_domain(self):
router = self._router(num_fused_shared_experts=1)
ids = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32)
weights = torch.tensor([[0.8, 0.2]]).repeat(4, 1)
solved = []
def solve(routed_ids):
self.assertEqual(routed_ids.tolist(), [[0], [1], [2], [3]])
solved.append(routed_ids.clone())
return torch.ones(4, 1)
def dispatch(routed_ids, info, probability=None):
self.assertTrue(bool(((routed_ids >= 0) & (routed_ids < 4)).all()))
return info.partial_logical_to_rank_dispatch_physical_map[routed_ids]
get_resources().lplb_solvers[0] = SimpleNamespace(solve=solve)
self.addCleanup(get_resources().lplb_solvers.pop, 0)
with (
get_exec().moe.override(ep_dispatch_algorithm="lp"),
patch.object(topk_module, "_is_cuda", True),
patch.object(topk_module, "topk_ids_logical_to_physical", dispatch),
):
_, info = router.routing_inputs()
output_ids, output_weights, _ = topk_module._post_process_topk_ids(
ids,
weights,
topk_config=router.topk_config,
router_logits=self.logits,
layer_id=0,
num_token_non_padded=torch.tensor(2),
expert_location_dispatch_info=info,
)
self.assertEqual(len(solved), 1)
self.assertEqual(output_ids.tolist(), [[2, 4], [0, 4], [-1, -1], [-1, -1]])
torch.testing.assert_close(output_weights[:2], weights[:2])
def test_lazy_lp_materialization_uses_its_owner_or_explicit_override(self):
"""Lazy dispatcher materialization must retain the layer's solver and capture slot."""
get_resources().expert_location_metadata = _placement(num_layers=8)
hidden = self.hidden.to(torch.bfloat16)
group = SimpleNamespace(all_gatherv=lambda tensors, sizes: tensors)
dispatcher = object.__new__(FlashinferDispatcher)
dispatcher.ep_size = 1
dispatcher.ep_rank = 0
with get_parallel().override(moe_ep_size=1, moe_ep_rank=0):
dispatcher.prefill_dispatcher = StandardDispatcher(
SimpleNamespace(
runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM,
num_experts=4,
num_local_experts=4,
num_fused_shared_experts=0,
)
)
for owner, override, expected_owner in (
(7, None, 7),
(0, None, 0),
(7, 0, 0),
(0, 7, 7),
):
with self.subTest(owner=owner, override=override):
solved = []
captured = {}
def solver(layer_id):
def solve(ids):
self.assertEqual(ids.tolist(), [[0, 1]] * 4)
solved.append(layer_id)
return torch.ones((4, 1))
return SimpleNamespace(solve=solve)
class Capture:
def capture(self, *, layer_id, topk_indices):
captured[layer_id] = topk_indices.clone()
def dispatch_probability(ids, info, probability):
return info.partial_logical_to_rank_dispatch_physical_map[ids]
with (
get_exec().moe.override(ep_dispatch_algorithm="lp"),
get_parallel().override(
tp_group=group, moe_ep_size=1, moe_ep_rank=0
),
patch.dict(
get_resources().lplb_solvers,
{0: solver(0), 7: solver(7)},
clear=True,
),
patch.object(get_resources(), "experts_capturer", Capture()),
patch.object(topk_module, "_is_cuda", True),
patch.object(
expert_location_dispatch,
"_topk_ids_logical_to_physical_probability",
dispatch_probability,
),
patch(
"sglang.srt.layers.moe.token_dispatcher.flashinfer.get_dp_global_num_tokens",
return_value=None,
),
):
router = self._router(
layer_id=owner, output_format=TopKOutputFormat.BYPASSED
)
output = router.forward_cuda(
hidden, self.logits, num_token_non_padded=torch.tensor(2)
)
self.assertEqual(len(output), 5)
if override is not None:
output = output.to_standard(layer_id=override)
dispatched = dispatcher._dispatch_prefill_allgather(hidden, output)
self.assertEqual(solved, [expected_owner])
self.assertEqual(set(captured), {expected_owner})
self.assertEqual(captured[expected_owner].tolist(), [[0, 1]] * 4)
self.assertEqual(
dispatched.topk_output.topk_ids.tolist(),
[[2, 0], [2, 0], [-1, -1], [-1, -1]],
)
self.assertIs(dispatched.hidden_states, hidden)
def test_empty_target_participates_in_lp_without_caller_layer_id(self):
class Solver:
def __init__(self):
self.participations = []
def solve(self, ids):
self.participations.append(tuple(ids.shape))
solver = Solver()
get_resources().lplb_solvers[0] = solver
self.addCleanup(get_resources().lplb_solvers.pop, 0)
target = self._router()
with draft_model_build_scope():
draft = self._router()
with get_parallel().override(tp_group=SimpleNamespace(world_size=1)):
output = target.empty_topk_output(torch.device("cpu"))
draft_output = draft.empty_topk_output(torch.device("cpu"))
self.assertEqual(output.topk_ids.shape, (0, 2))
self.assertEqual(draft_output.topk_ids.shape, (0, 2))
self.assertEqual(solver.participations, [(0, 2)])
def test_compile_reads_current_buffer(self):
router = self._router(is_draft=True)
batch = _batch(3)
def mask_rows(x):
bound, _ = router.routing_inputs()
return x * (torch.arange(x.shape[0]) < bound)
compiled = torch.compile(mask_rows, backend="eager", fullgraph=True)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=batch
):
self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 1.0, 1.0, 0.0])
batch.num_token_non_padded.fill_(1)
self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 0.0, 0.0, 0.0])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,342 @@
"""GGUF expert kernels drop inactive routes during padded graph replay."""
import unittest
from contextlib import nullcontext
from unittest.mock import patch
import numpy as np
import torch
from gguf import GGMLQuantizationType, dequantize, quantize
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend
from sglang.srt.layers.quantization import gguf as gguf_runtime
from sglang.srt.layers.quantization.gguf import (
GGUFConfig,
GGUFMoEMethod,
fused_moe_gguf,
ggml_moe_get_block_size,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=30, stage="base-b", runner_config="1-gpu-small")
class TestGgufInactiveRoutes(CustomTestCase):
def setUp(self):
self.enterContext(
get_context().override_server_args(model_path="dummy", device="cuda")
)
self.enterContext(
get_flags().moe.override(
runner_backend=MoeRunnerBackend.AUTO, a2a_backend=MoeA2ABackend.NONE
)
)
self.enterContext(
get_parallel().override(
tp_size=1,
tp_rank=0,
attn_tp_size=1,
attn_tp_rank=0,
attn_dp_size=1,
attn_cp_size=1,
tp_group=None,
moe_tp_size=1,
moe_tp_rank=0,
moe_ep_size=1,
moe_ep_rank=0,
)
)
@staticmethod
def _weights(quant_type):
rng = np.random.default_rng(491)
shape13, shape2 = (4, 512, 256), (4, 256, 256)
packed, dense = [], []
for shape in (shape13, shape2):
weight = rng.normal(0, 0.03, shape).astype(np.float32)
qweight = quantize(weight, quant_type)
packed.append(torch.from_numpy(qweight.copy()).cuda())
dense.append(torch.from_numpy(dequantize(qweight, quant_type)).cuda())
return packed, dense
@staticmethod
def _method(packed, quant_type, *, global_experts=4):
method = GGUFMoEMethod(GGUFConfig())
layer = torch.nn.Module()
method.create_weights(
layer,
num_experts=4,
hidden_size=256,
intermediate_size_per_partition=256,
params_dtype=torch.float16,
)
for param, weight in zip((layer.w13_qweight, layer.w2_qweight), packed):
param.materialize(weight.shape, device=weight.device, dtype=weight.dtype)
param.data.copy_(weight)
layer.w13_qweight_type.weight_type = int(quant_type)
layer.w2_qweight_type.weight_type = int(quant_type)
config = MoeRunnerConfig(
num_experts=global_experts,
num_local_experts=4,
num_fused_shared_experts=0,
hidden_size=256,
intermediate_size_per_partition=256,
top_k=2,
)
method.create_moe_runner(layer, config)
config.runner_backend = method.get_moe_runner_backend(layer)
return method, layer, StandardDispatcher(config)
@staticmethod
def _q8_1_matmul(inputs, weight, packed_weight, quant_type):
# GGUF GEMMs quantize each 32 input values to Q8_1, including an FP16
# scale and input sum. CUDA roundf rounds halfway values away from zero.
blocks = inputs.float().unflatten(-1, (-1, 32))
scale = blocks.abs().amax(dim=-1, keepdim=True) / 127
denominator = torch.where(scale == 0, 1.0, scale)
codes = blocks.sign() * torch.floor(blocks.abs() / denominator + 0.5)
quantized = codes * scale.half().float()
result = torch.einsum(
"...bi,...nbi->...n", quantized, weight.unflatten(-1, (-1, 32))
)
if quant_type == GGMLQuantizationType.Q4_0:
# Q4_0 subtracts its zero point using the original FP16 input sum,
# rather than summing the dequantized Q8_1 codes.
weight_scale = (
packed_weight.unflatten(-1, (-1, 18))[..., :2]
.contiguous()
.view(torch.float16)
.squeeze(-1)
.float()
)
sum_error = quantized.sum(dim=-1) - blocks.sum(dim=-1).half().float()
result += 8 * torch.einsum("...b,...nb->...n", sum_error, weight_scale)
return result.to(inputs.dtype)
@classmethod
def _reference(cls, hidden, logits, dense, packed, quant_type):
weights, ids = logits.float().softmax(dim=-1).topk(2, dim=-1)
projected = cls._q8_1_matmul(
hidden.unsqueeze(1), dense[0][ids], packed[0][ids], quant_type
)
gate, up = projected.float().chunk(2, dim=-1)
activated = (torch.nn.functional.silu(gate) * up).to(hidden.dtype)
down = cls._q8_1_matmul(activated, dense[1][ids], packed[1][ids], quant_type)
weighted = (down.float() * weights.unsqueeze(-1)).to(hidden.dtype)
return weighted.sum(dim=1).float()
def test_block_size_metadata_without_tensor_arguments(self):
# Runtime metadata must work without a Tensor-dispatched wheel operator.
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
with self.subTest(quant_type=quant_type):
self.assertIn(ggml_moe_get_block_size(int(quant_type)), (4, 8))
@torch.inference_mode()
def test_padded_graph_replay_matches_unpadded_experts(self):
"""Changing the live count must zero dropped GGUF routes on every replay."""
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
packed, dense = self._weights(quant_type)
for tokens, partial in ((64, 48), (96, 71)):
with self.subTest(quant_type=quant_type, tokens=tokens):
self._check_graph(packed, dense, quant_type, tokens, partial)
def _check_graph(self, packed, dense, quant_type, tokens, partial):
method, layer, dispatcher = self._method(packed, quant_type)
topk = TopK(
top_k=2, layer_id=0, renormalize=False, allow_routed_experts_capture=False
)
topk.bind_runner_backend(method.get_moe_runner_backend(layer))
generator = torch.Generator(device="cuda").manual_seed(tokens)
base_hidden = torch.randn(
tokens, 256, generator=generator, device="cuda"
).half()
base_logits = torch.randn(tokens, 4, generator=generator, device="cuda")
logits = base_logits.clone()
hidden = base_hidden.clone()
live_count = torch.tensor(tokens, device="cuda", dtype=torch.int32)
batch = ForwardBatch(
forward_mode=ForwardMode.EXTEND,
batch_size=1,
input_ids=torch.zeros(tokens, device="cuda", dtype=torch.int64),
req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64),
seq_lens=torch.tensor([tokens], device="cuda", dtype=torch.int32),
out_cache_loc=torch.arange(tokens, device="cuda"),
seq_lens_sum=tokens,
positions=torch.arange(tokens, device="cuda"),
num_token_non_padded=live_count,
global_num_token_non_padded=live_count,
)
def run():
with forward_context(
ForwardContext(attn_backend=None), forward_batch=batch
):
routes = topk.forward_cuda(hidden, logits)
dispatched = dispatcher.dispatch(hidden, routes)
combined = method.apply(layer, dispatched)
return dispatcher.combine(combined), routes.topk_ids
native_outputs = []
native_mmq = gguf_runtime.ggml_moe_a8
native_align = gguf_runtime.moe_align_block_size
def poison_alignment(*args, **kwargs):
sorted_ids, expert_ids, count = native_align(*args, **kwargs)
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * 4
)
expert_ids.masked_fill_(block_starts >= count, 4)
return sorted_ids, expert_ids, count
def record_mmq_outputs(*args, **kwargs):
if not torch.cuda.is_current_stream_capturing():
# Fail before native dispatch if a slack block can index expert E.
self.assertTrue(bool((args[3] < args[1].shape[0]).all()))
result = native_mmq(*args, **kwargs)
if torch.cuda.is_current_stream_capturing():
native_outputs.append(result)
return result
# Observe the real provider's returned buffers without replacing its
# computation. Poisoning them makes a skipped MMQ store deterministic.
observe_mmq = (
patch.object(gguf_runtime, "ggml_moe_a8", record_mmq_outputs)
if tokens > 64
else nullcontext()
)
with (
observe_mmq,
patch.object(gguf_runtime, "moe_align_block_size", poison_alignment),
):
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
run()
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output, ids = run()
self.assertEqual(len(native_outputs), 2 if tokens > 64 else 0)
# Dropped slots must overwrite poisoned native intermediates rather
# than inherit any previous live route or allocator contents.
for live in (tokens, partial, 0, partial // 2, tokens):
hidden.copy_(base_hidden)
hidden[live:].fill_(float("nan"))
logits.copy_(base_logits)
logits[live:].fill_(float("nan"))
live_count.fill_(live)
output.fill_(float("nan"))
for intermediate in native_outputs:
intermediate.fill_(float("nan"))
graph.replay()
for intermediate in native_outputs:
self.assertTrue(torch.isfinite(intermediate).all().item())
inactive = intermediate.view(tokens, 2, -1)[live:]
self.assertEqual(torch.count_nonzero(inactive).item(), 0)
self.assertTrue(torch.isfinite(output).all().item())
self.assertTrue((ids[live:] == -1).all().item())
self.assertEqual(torch.count_nonzero(output[live:]).item(), 0)
if live:
expected = self._reference(
base_hidden[:live], base_logits[:live], dense, packed, quant_type
)
self.assertGreater(expected.abs().max().item(), 1e-3)
torch.testing.assert_close(
output[:live].float(), expected, atol=2e-3, rtol=0.06
)
@torch.inference_mode()
def test_nonlocal_routes_preserve_local_contributions(self):
"""A nonlocal slot must not erase the same token's valid expert result."""
packed, dense = self._weights(GGMLQuantizationType.Q4_0)
with get_parallel().override(
tp_size=2,
tp_rank=1,
attn_tp_size=2,
attn_tp_rank=1,
moe_ep_size=2,
moe_ep_rank=1,
):
method, layer, dispatcher = self._method(
packed, GGMLQuantizationType.Q4_0, global_experts=8
)
hidden = torch.full((3, 256), 0.25, device="cuda", dtype=torch.float16)
ids = torch.tensor([[4, 1], [-1, 5], [0, 2]], device="cuda", dtype=torch.int32)
weights = torch.tensor(
[[0.7, float("nan")], [float("nan"), 0.6], [float("nan"), float("nan")]],
device="cuda",
)
hidden[2].fill_(float("nan"))
routes = StandardTopKOutput(weights, ids, None)
output = dispatcher.combine(
method.apply(layer, dispatcher.dispatch(hidden, routes))
)
self.assertTrue(torch.isfinite(output).all().item())
self.assertEqual(torch.count_nonzero(output[2]).item(), 0)
for token, expert, scale in ((0, 0, 0.7), (1, 1, 0.6)):
projected = self._q8_1_matmul(
hidden[token],
dense[0][expert],
packed[0][expert],
GGMLQuantizationType.Q4_0,
)
gate, up = projected.float().chunk(2)
activated = (torch.nn.functional.silu(gate) * up).half()
expected = (
(
self._q8_1_matmul(
activated,
dense[1][expert],
packed[1][expert],
GGMLQuantizationType.Q4_0,
).float()
* scale
)
.half()
.float()
)
torch.testing.assert_close(
output[token].float(), expected, atol=2e-3, rtol=0.06
)
@torch.inference_mode()
def test_unquantized_fallback_drops_negative_ids(self):
"""The slow path must not alias a dropped route to the last expert."""
generator = torch.Generator(device="cuda").manual_seed(17)
w13 = torch.randn(2, 32, 16, generator=generator, device="cuda").half() / 16
w2 = torch.randn(2, 16, 16, generator=generator, device="cuda").half() / 16
hidden = torch.ones(2, 16, device="cuda", dtype=torch.float16)
hidden[1].fill_(float("nan"))
ids = torch.tensor([[0, -1], [-1, -1]], device="cuda", dtype=torch.int32)
weights = torch.tensor(
[[1.0, float("nan")], [float("nan"), float("nan")]], device="cuda"
)
output = fused_moe_gguf(
hidden,
w13,
w2,
weights,
ids,
GGMLQuantizationType.F16,
GGMLQuantizationType.F16,
"silu",
)
projected = hidden[0] @ w13[0].t()
gate, up = projected.chunk(2)
expected = (torch.nn.functional.silu(gate) * up) @ w2[0].t()
self.assertTrue(torch.isfinite(output).all().item())
self.assertEqual(torch.count_nonzero(output[1]).item(), 0)
torch.testing.assert_close(output[0], expected, atol=1e-3, rtol=1e-3)
if __name__ == "__main__":
unittest.main()
@@ -196,7 +196,7 @@ class _MockDispatchOutput:
self.hidden_states = hidden_states
self.topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=TopKConfig(top_k=top_k, renormalize=True),
)
@@ -189,6 +189,7 @@ def _run_marlin_flow(
fused_tails=0,
marlin_is_ep=[],
marlin_output_ids=[],
marlin_output_initialized=[],
align_num_experts=[],
schedule=[],
zeroed=[],
@@ -248,6 +249,15 @@ def _run_marlin_flow(
return _ORIGINAL_TENSOR_ZERO(tensor, *args, **kwargs)
monkeypatch.setattr(torch.Tensor, "zero_", tracked_zero)
original_empty = torch.empty
def poisoned_empty(*args, **kwargs):
result = original_empty(*args, **kwargs)
if result.is_floating_point():
result.fill_(float("nan"))
return result
monkeypatch.setattr(torch, "empty", poisoned_empty)
def merged_experts_fused_moe_lora_add(**kwargs):
intermediate = kwargs.get("intermediate_buffer")
@@ -298,11 +308,13 @@ def _run_marlin_flow(
def fake_marlin_gemm(_x, output, *_args, **kwargs):
calls.marlin_is_ep.append(kwargs["is_ep"])
calls.marlin_output_ids.append(id(output))
calls.marlin_output_initialized.append(
bool(torch.isfinite(output).all() and torch.count_nonzero(output) == 0)
)
calls.schedule.append(
("marlin", stream_state["current"].name, len(calls.marlin_is_ep))
)
# ``fill_``, never ``zero_``: the zero-fill contracts are asserted by
# watching ``Tensor.zero_`` calls, so the fakes must not forge one.
# Inspect initialization before the fake GEMM writes its result.
output.fill_(0)
return output
@@ -410,22 +422,14 @@ def _run_marlin_flow(
def test_ep_uses_local_alignment_and_skips_nonlocal_marlin_blocks(monkeypatch):
"""Guards expert-parallel index localization and both accumulator clears;
reds when ``moe_align_block_size`` gets the global expert count, ``is_ep``
is dropped on a Marlin GEMM, or an accumulator is left uninitialized."""
"""EP alignment stays local and the down accumulator starts finite and zero."""
calls = _run_marlin_flow(monkeypatch, tokens=16, ep=True, slots=2, rank=128)
assert calls.align_num_experts == [1]
assert calls.marlin_is_ep == [True, True]
down_gemm_output = calls.marlin_output_ids[1]
assert [item[2] for item in calls.zeroed] == [down_gemm_output]
zero_index = calls.schedule.index(("zero", "main", down_gemm_output))
down_gemm_index = [
index for index, item in enumerate(calls.schedule) if item[0] == "marlin"
][1]
assert zero_index < down_gemm_index
assert calls.marlin_output_initialized[1]
down_stage = [
call for call in calls.merged if call["stage"] == "all" and call["shared_b"]
@@ -0,0 +1,350 @@
"""Executor-owned routing bounds survive retained callbacks and padded CPU replay."""
import importlib
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.environ import envs
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.dp_attention import DpPaddingMode
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.layers.moe import topk as topk_module
from sglang.srt.layers.moe.topk import TopK, fused_topk_torch_native
from sglang.srt.layers.moe.utils import initialize_moe_config
from sglang.srt.model_executor.cpu_graph_runner import CPUGraphRunner
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
ForwardMode,
PPProxyTensors,
enable_num_token_non_padded,
)
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
DecodeCudaGraphRunner,
)
from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import (
BreakableCudaGraphBackend,
)
from sglang.srt.runtime_context import get_context, get_parallel
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
breakable_graph = importlib.import_module(
"sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph"
)
breakable_backend = importlib.import_module(
"sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend"
)
def _batch(size, live, *, prepared=True):
batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
batch_size=size,
input_ids=torch.arange(size),
req_pool_indices=torch.arange(size),
seq_lens=torch.ones(size, dtype=torch.int32),
out_cache_loc=torch.arange(size),
seq_lens_sum=size,
positions=torch.arange(size),
capture_hidden_mode=CaptureHiddenMode.NULL,
)
batch.global_num_token_non_padded = torch.tensor(live, dtype=torch.int32)
batch.global_num_token_non_padded_cpu = live
batch.num_token_non_padded = batch.global_num_token_non_padded if prepared else None
batch.global_dp_buffer_len = size
batch.dp_padding_mode = DpPaddingMode.SUM_LEN
return batch
def _attention_backend():
return SimpleNamespace(
init_forward_metadata=lambda batch: None,
init_forward_metadata_out_graph=lambda batch, **kwargs: None,
init_forward_metadata_in_graph=lambda batch: None,
shared_read_ends=lambda mode: SharedReadEnds.UNKNOWN,
)
class _RoutingModel:
shared_experts_config = SharedExpertsConfig()
def __init__(self, *, proxy_output):
self.topk = TopK(
2,
layer_id=0,
is_draft=True,
custom_routing_function=fused_topk_torch_native,
)
self.proxy_output = proxy_output
self.seen = []
self.fail = False
def forward(self, input_ids, positions, batch):
rows = len(input_ids)
bound, _ = self.topk.routing_inputs()
ids = self.topk.forward_native(
torch.ones((rows, 2)),
torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(rows, 1),
).topk_ids
self.seen.append(
(batch, bound, get_forward_context().attn_backend, ids.clone())
)
if self.fail:
raise RuntimeError("replayed model failed")
if self.proxy_output:
return PPProxyTensors({"routing_ids": ids})
return LogitsProcessorOutput(next_token_logits=ids.float())
class _CpuStream:
def wait_stream(self, other):
pass
class _NoDeviceGraph:
"""Only accelerator graph recording/replay is replaced; eager breaks remain real."""
def capture_begin(self, **kwargs):
pass
def capture_end(self):
pass
def replay(self):
pass
class TestGraphRoutingContext(CustomTestCase):
def setUp(self):
super().setUp()
config = get_context().override_server_args(
enable_eplb=False, ep_dispatch_algorithm=None, debug_cuda_graph=True
)
config.install()
self.addCleanup(config.restore)
initialize_moe_config()
topology = get_parallel().override(
tp_size=4,
moe_ep_size=4,
attn_dp_size=1,
attn_cp_size=1,
attn_tp_size=4,
attn_tp_rank=0,
tp_group=SimpleNamespace(barrier=lambda: None, world_size=4),
)
topology.__enter__()
self.addCleanup(topology.__exit__, None, None, None)
for name in ("_is_cuda", "_is_hip", "_use_aiter"):
scope = patch.object(topk_module, name, False)
scope.start()
self.addCleanup(scope.stop)
def _model_runner(self, *, device, model):
runner = ModelRunner.__new__(ModelRunner)
runner.device = device
runner.attn_backend = _attention_backend()
runner.hisparse_coordinator = None
runner.device_timer = None
runner.is_draft_worker = False
runner.spec_algorithm = SpeculativeAlgorithm.NONE
runner.capture_tail_hooks = []
runner.canary_manager = None
runner.model = model
return runner
def _debug_runner(self):
model = _RoutingModel(proxy_output=True)
model_runner = self._model_runner(device="cuda", model=model)
runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner)
runner.model_runner = model_runner
runner.captured_req_width = 1
runner.ragged_verify_mode = False
runner.enable_pdmux = False
runner.pp_size = 1
runner.dllm_uses_input_embeds = False
runner.in_graph_metadata_prep_done = None
runner.is_dllm = False
static = _batch(4, 4)
runner.attn_backend = captured_backend = _attention_backend()
runner.capture_prepare = lambda *args, **kwargs: (
static,
captured_backend,
None,
)
runner.tbo_plugin = SimpleNamespace(
capture_one_batch_size=lambda *args, **kw: None
)
runner.deepep_adapter = SimpleNamespace(capture=lambda **kwargs: None)
stream = _CpuStream()
runner.device_module = SimpleNamespace(
Stream=_CpuStream,
current_stream=lambda *args: stream,
synchronize=lambda: None,
is_current_stream_capturing=lambda: False,
)
runner.backend = BreakableCudaGraphBackend(runner, debug_eager=True)
runner.can_run_graph = lambda batch: True
def load_batch(batch, pp_proxy_tensors):
# Substitute only the accelerator input copies/shape selection.
static.num_token_non_padded.copy_(batch.global_num_token_non_padded)
runner.raw_bs = runner.raw_num_token = batch.batch_size
runner.bs = 4
runner._replay_graph_key = runner._make_graph_key(4)
runner.load_batch = load_batch
model_runner.decode_cuda_graph_runner = runner
for target, name, replacement in (
(torch.cuda, "CUDAGraph", _NoDeviceGraph),
(breakable_graph, "get_device_module", lambda: runner.device_module),
(breakable_backend, "graph_pool_capture_scope", nullcontext),
(breakable_backend, "graph_pool_replay_scope", nullcontext),
):
scope = patch.object(target, name, replacement)
scope.start()
self.addCleanup(scope.stop)
with envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE.override(False):
runner.capture_one_shape(4, model.forward)
self.assertEqual(
len(runner.backend._graphs[runner._make_graph_key(4)]._break_fns), 1
)
return model_runner, model, static, captured_backend
def _cpu_runner(self):
model = _RoutingModel(proxy_output=False)
model_runner = self._model_runner(device="cpu", model=model)
runner = CPUGraphRunner.__new__(CPUGraphRunner)
runner.model_runner = model_runner
runner.is_encoder_decoder = False
runner.capture_hidden_mode = CaptureHiddenMode.NULL
runner.capture_bs = [4]
runner.captured_req_width = 1
runner.seq_len_fill_value = 1
runner.graphs = {4: model.forward}
runner.graphs_cross = {}
static = _batch(4, 4)
runner.captured_forward_batches = {4: static}
runner.can_run_graph = lambda batch: True
model_runner.decode_cuda_graph_runner = runner
return model_runner, model, static, model_runner.attn_backend
def _capture_cpu_runner(self):
model_runner, model, _, backend = self._cpu_runner()
runner = model_runner.decode_cuda_graph_runner
runner.capture_bs = [1, 4]
runner.capture_forward_mode = ForwardMode.DECODE
runner.captured_forward_batches = {}
runner.captured_forward_batches_cross = {}
runner.input_ids = torch.arange(4)
runner.req_pool_indices = torch.arange(4)
runner.seq_lens = torch.ones(4, dtype=torch.int32)
runner.out_cache_loc = torch.arange(4)
runner.positions = torch.arange(4)
runner.mrope_positions = torch.zeros((3, 4), dtype=torch.int64)
runner.num_token_non_padded = torch.zeros(1, dtype=torch.int64)
model_runner.kv_index_translator = SimpleNamespace(
bind_runner_slots=lambda batch: None
)
backend.init_forward_metadata_capture_cpu_graph = lambda *args: None
for size in reversed(runner.capture_bs):
runner.graphs[size], _ = runner.capture_one_batch_size(
size, model.forward, skip_cross_attention=True
)
return runner, model, backend
def test_cpu_capture_without_tracking_keeps_live_routes(self):
"""Padded CPU replays keep every live route when count tracking is disabled."""
with get_parallel().override(
tp_size=1,
moe_ep_size=1,
attn_tp_size=1,
tp_group=SimpleNamespace(barrier=lambda: None, world_size=1),
):
self.assertFalse(enable_num_token_non_padded())
runner, model, backend = self._capture_cpu_runner()
for size in (3, 4, 2, 1):
with self.subTest(size=size):
incoming = _batch(size, size, prepared=False)
incoming.global_num_token_non_padded = None
with forward_context(
ForwardContext(attn_backend=backend), forward_batch=incoming
):
eager = model.forward(
incoming.input_ids, incoming.positions, incoming
)
output = runner.execute(incoming)
torch.testing.assert_close(
output.next_token_logits, eager.next_token_logits
)
prepared, bound, _, ids = model.seen[-1]
self.assertIsNone(bound)
self.assertTrue((ids >= 0).all())
self.assertEqual(prepared.batch_size, 4 if size > 1 else 1)
self.assertFalse(has_forward_context())
for captured in runner.captured_forward_batches.values():
self.assertIsNone(captured.num_token_non_padded)
def _assert_replay(self, model_runner, model, static, backend):
persistent_bound = static.num_token_non_padded
for live in (3, 1, 0):
incoming = _batch(live, live, prepared=False)
model_runner._forward_raw(incoming, None)
batch, bound, active_backend, ids = model.seen[-1]
self.assertIs(batch, static)
self.assertIs(bound, persistent_bound)
self.assertEqual(int(bound), live)
self.assertIs(active_backend, backend)
self.assertEqual(ids[:live].tolist(), [[0, 1]] * live)
self.assertEqual(ids[live:].tolist(), [[-1, -1]] * (4 - live))
self.assertIsNone(incoming.num_token_non_padded)
self.assertFalse(has_forward_context())
def test_debug_saved_forward_rebinds_static_batch(self):
self._assert_replay(*self._debug_runner())
def test_cpu_replay_rebinds_actual_prepared_batch(self):
self._assert_replay(*self._cpu_runner())
def _assert_failure_restores_parent(self, model_runner, model, static, backend):
parent = _batch(1, 1)
with forward_context(
ForwardContext(attn_backend=object()), forward_batch=parent
):
previous = get_forward_context()
model.fail = True
with self.assertRaisesRegex(RuntimeError, "replayed model failed"):
model_runner._forward_raw(_batch(3, 3, prepared=False), None)
self.assertIs(model.seen[-1][0], static)
self.assertIs(model.seen[-1][1], static.num_token_non_padded)
self.assertIs(model.seen[-1][2], backend)
self.assertIs(get_forward_context(), previous)
self.assertIs(
previous.moe_num_token_non_padded(), parent.num_token_non_padded
)
self.assertFalse(has_forward_context())
def test_debug_saved_forward_failure_restores_parent(self):
self._assert_failure_restores_parent(*self._debug_runner())
def test_cpu_replay_failure_restores_parent(self):
self._assert_failure_restores_parent(*self._cpu_runner())
if __name__ == "__main__":
unittest.main()
@@ -22,12 +22,13 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
import unittest
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from types import SimpleNamespace
from unittest.mock import patch
from unittest.mock import Mock, patch
import torch
import sglang.srt.models.deepseek_v4_replay_tail as replay_tail
from sglang.srt.layers.moe import utils as moe_utils
from sglang.srt.layers.moe.utils import MoeA2ABackend
from sglang.srt.model_executor.forward_batch_info import (
@@ -37,6 +38,15 @@ from sglang.srt.model_executor.forward_batch_info import (
compute_local_num_token_non_padded_cpu,
enable_num_token_non_padded,
)
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
RowBucketGraphRunner,
)
from sglang.srt.runtime_context import get_flags, get_parallel
from sglang.test.communicator_patch import patch_communicator
from sglang.test.test_utils import CustomTestCase
@@ -254,5 +264,185 @@ class TestEnableNumTokenNonPadded(CustomTestCase):
self.assertIs(enable_num_token_non_padded(), expected)
class TestRowBucketMoeBound(CustomTestCase):
def test_capture_owns_the_moe_bound_and_restores_the_outer_context(self):
"""A bucket's recorded routing must retain its own mutable count buffer."""
for active_context in (False, True):
with (
self.subTest(active_context=active_context),
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
sparse_moe_input("local"),
):
outer_batch = _forward_batch(sharded=False, local=3, glob=3)
outer = ForwardContext(attn_backend=None)
scope = (
forward_context(outer, forward_batch=outer_batch)
if active_context
else nullcontext()
)
observed = []
def body(inputs, capture_batch):
if active_context:
bound = get_forward_context().moe_num_token_non_padded()
self.assertIs(bound, capture_batch.num_token_non_padded)
observed.append(bound)
else:
self.assertFalse(has_forward_context())
return inputs
def record(body, rows, count, batch, start, free_before):
body()
return SimpleNamespace(batch=(), num_token_non_padded=count)
runner = RowBucketGraphRunner(row_step=8, max_rows=8, label="test")
with (
scope,
patch.object(torch.cuda, "mem_get_info", return_value=(1, 2)),
patch.object(runner, "_record", side_effect=record),
):
graph = runner._capture(
8,
3,
[torch.zeros(8, 1)],
outer_batch,
body,
lambda batch, inputs: nullcontext(),
)
graph.num_token_non_padded.fill_(7)
outer_batch.num_token_non_padded.fill_(1)
if active_context:
self.assertEqual(int(observed[0]), 7)
self.assertIs(
get_forward_context().moe_num_token_non_padded(),
outer_batch.num_token_non_padded,
)
self.assertFalse(has_forward_context())
class TestDecoderReplayTailMoeBound(CustomTestCase):
# A row graph replaces LOCAL only, so even equal-valued or aliased GLOBAL
# buffers cannot establish that the captured getter reads its own count.
_CASES = (
("local", False, 3, 7, False, True),
("global", True, 3, 7, False, False),
("equal_global", True, 3, 3, False, False),
("aliased_local", False, 3, 3, True, False),
("aliased_global", True, 3, 3, True, False),
("unbounded_global", True, 3, None, False, True),
("absent_local", True, None, 7, False, True),
)
@staticmethod
def _batch_and_tail(sharded, local, glob, alias):
batch = _forward_batch(
sharded=sharded, local=local, glob=glob, forward_mode=ForwardMode.EXTEND
)
batch.input_ids = torch.arange(4)
if alias:
batch.global_num_token_non_padded = batch.num_token_non_padded
tail = SimpleNamespace(
positions=torch.arange(3), cp_metadata=None, rows=lambda t: t[:3]
)
return batch, tail
def test_eager_prefill_declines_a_separate_moe_count_buffer(self):
for label, sharded, local, glob, alias, expected in self._CASES:
with (
self.subTest(bound=label),
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
sparse_moe_input("ffn"),
):
batch, tail = self._batch_and_tail(sharded, local, glob, alias)
original_global = batch.global_num_token_non_padded
tail_layers = replay_tail.DecoderReplayTail.__new__(
replay_tail.DecoderReplayTail
)
tail_layers.runner = SimpleNamespace(
ready=Mock(), warm=True, bucket_rows=lambda n: 8
)
with (
patch.object(
replay_tail, "get_is_capture_mode", return_value=False
),
patch.object(
replay_tail,
"get_global_expert_distribution_recorder",
return_value=SimpleNamespace(recording=False),
),
patch(
"sglang.srt.layers.attention.dsv4.unified_kv_gate.is_unified_kv_triton",
return_value=False,
),
):
use_graph, dp_rows = tail_layers.plan(
tail,
forward_batch=batch,
num_tokens=4,
tail_in_graph=False,
late_dp_counts=None,
)
self.assertEqual(use_graph, expected)
self.assertIsNone(dp_rows)
self.assertIs(batch.global_num_token_non_padded, original_global)
def test_prefill_graph_break_keeps_a_separate_moe_count_eager(self):
for label, sharded, local, glob, alias, expected in self._CASES:
with (
self.subTest(bound=label),
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
sparse_moe_input("ffn"),
):
batch, tail = self._batch_and_tail(sharded, local, glob, alias)
original_global = batch.global_num_token_non_padded
output = torch.tensor([11])
tail_layers = SimpleNamespace(
runner=SimpleNamespace(warm=True, bucket_rows=lambda n: 8),
_can_capture_moe_bound=(
replay_tail.DecoderReplayTail._can_capture_moe_bound
),
run=Mock(return_value=(output, None, [])),
_run_layers_eagerly=Mock(return_value=(output, None, [])),
_bridge_outputs=lambda rows, outputs: tuple(outputs),
)
backend = SimpleNamespace(
forward_metadata=SimpleNamespace(late_layer_tail=tail),
exit_late_layer_tail=Mock(),
)
with (
patch.object(replay_tail, "get_attn_backend", return_value=backend),
patch.object(
replay_tail,
"get_tc_piecewise_forward_context",
return_value=SimpleNamespace(forward_batch=batch),
),
patch.object(
replay_tail,
"_enter_eager_tail_in_prefill_graph",
return_value=(None, batch),
),
get_flags().dp.override(capturing_prefill_graph=False),
):
outputs = replay_tail._tail_layers_in_prefill_graph(
tail_layers,
lambda tensors: tensors,
False,
4,
None,
False,
[torch.zeros(4, 1)],
torch.arange(4),
batch.input_ids,
batch.input_ids,
)
self.assertIs(outputs[0], output)
self.assertEqual(tail_layers.run.call_count, expected)
self.assertEqual(
tail_layers._run_layers_eagerly.call_count, not expected
)
backend.exit_late_layer_tail.assert_called_once_with(None, batch)
self.assertIs(batch.global_num_token_non_padded, original_global)
if __name__ == "__main__":
unittest.main()
@@ -44,6 +44,11 @@ DRAFT_SUFFIX = re.compile(r"(NextN|MTP|DSpark|DFlash|Standalone)$")
def _target_of(arch: str, archs: dict):
"""The arch a draft entry class drafts for, if it is named after it."""
# Bailing shares its NextN entry class across generations. Its loader
# fusion hooks belong to V3; legacy checkpoints return a fused count of 0
# (covered by TestBailingMoeV3Gate in test_shared_experts_fusion_gates.py).
if arch == "BailingMoeForCausalLMNextN":
return "BailingMoeV3ForCausalLM"
match = DRAFT_SUFFIX.search(arch)
if not match:
return None
@@ -11,7 +11,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
from sglang.srt.models.longcat_flash import LongcatFlashRouter # noqa: E402
from sglang.srt.models.longcat_flash import ( # noqa: E402
LongcatFlashMoE,
LongcatFlashRouter,
)
from sglang.srt.runtime_context import get_context, get_exec, get_parallel # noqa: E402
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
@@ -25,6 +29,37 @@ def _longcat_config(hidden_size, n_routed_experts, *, router_bias=False):
class TestLongcatFlashRouterHpcGemm(CustomTestCase):
def test_zero_experts_reject_placement_before_loading_expert_weights(self):
"""Identity expert IDs cannot index routed-only placement tables."""
config = SimpleNamespace(
routed_scaling_factor=1.0,
n_routed_experts=4,
moe_topk=2,
zero_expert_num=2,
zero_expert_type="identity",
rounter_params_dtype="float32",
hidden_act="silu",
)
with get_context().override_server_args(model_path="dummy", device="cpu"):
for algorithm in ("static", "dynamic", "fake", "lp"):
with (
self.subTest(algorithm=algorithm),
get_exec().moe.override(ep_dispatch_algorithm=algorithm),
self.assertRaisesRegex(ValueError, "LongCat zero experts"),
):
LongcatFlashMoE(config=config, layer_id=0)
with (
get_exec().moe.override(ep_dispatch_algorithm=None),
get_parallel().override(tp_size=1),
patch(
"sglang.srt.models.longcat_flash.LongcatFlashRouter",
side_effect=RuntimeError("placement admitted"),
),
self.assertRaisesRegex(RuntimeError, "placement admitted"),
):
LongcatFlashMoE(config=config, layer_id=0)
def _assert_dispatches_to_hpc_gemm(
self,
*,