[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)
Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
co-authored by
Aditya Suresh
parent
533020b500
commit
299a75ac5d
@@ -43,7 +43,7 @@ jobs:
|
||||
with:
|
||||
path: python/sglang/kernels/aot/dist/
|
||||
merge-multiple: true
|
||||
pattern: wheel-python3.10-cuda*
|
||||
pattern: wheel-python3.11-cuda*
|
||||
|
||||
- uses: ./.github/actions/download-rust-ext
|
||||
with:
|
||||
|
||||
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
|
||||
|
||||
m.def("ggml_moe_get_block_size(int type) -> int");
|
||||
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
|
||||
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
|
||||
|
||||
/*
|
||||
* From csrc/mamba
|
||||
|
||||
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
|
||||
const int nrows_y,
|
||||
const int nrows_dst,
|
||||
const int top_k) {
|
||||
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
|
||||
|
||||
const int blocks_per_row_x = ncols_x / qk;
|
||||
const int blocks_per_col_y = nrows_y / QK8_1;
|
||||
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
|
||||
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
|
||||
}
|
||||
|
||||
const int exp_idx = expert_ids[blockIdx.y];
|
||||
if (exp_idx > 255 || exp_idx < 0) return;
|
||||
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
|
||||
if (exp_idx < 0) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
const int col_dst = token_offs[j / nwarps];
|
||||
if (col_dst >= ncols_dst) continue;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
const auto row_dst = row_dst_0 + threadIdx.x + i;
|
||||
if (row_dst < nrows_dst) {
|
||||
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (exp_idx > 255) return;
|
||||
|
||||
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
|
||||
const block_q8_1* y = (const block_q8_1*)(vy);
|
||||
|
||||
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
|
||||
if (row >= nrows) {
|
||||
return;
|
||||
}
|
||||
if (expert < 0) {
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
||||
|
||||
@@ -383,14 +383,17 @@ __global__ void Marlin(
|
||||
|
||||
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
int parallel = num_tokens_past_padded / moe_block_size;
|
||||
int num_valid_blocks = parallel;
|
||||
if constexpr (kIsEP) {
|
||||
for (int i = 0; i < parallel; i++) {
|
||||
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
|
||||
}
|
||||
// Every warp counts the same inactive blocks without a serial per-thread scan.
|
||||
int num_invalid_blocks = 0;
|
||||
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
|
||||
num_invalid_blocks += expert_ids_ptr[i] == -1;
|
||||
}
|
||||
int num_invalid_blocks = parallel - num_valid_blocks;
|
||||
parallel = num_valid_blocks;
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
|
||||
}
|
||||
parallel -= num_invalid_blocks;
|
||||
if (parallel == 0) return;
|
||||
|
||||
int k_tiles = prob_k / 16 / thread_k_blocks;
|
||||
int n_tiles = prob_n / 16 / thread_n_blocks;
|
||||
@@ -518,23 +521,18 @@ __global__ void Marlin(
|
||||
if (par_id >= parallel) return;
|
||||
|
||||
old_expert_id = expert_id;
|
||||
if constexpr (kIsEP) {
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
skip_count--;
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
expert_id = expert_ids_ptr[block_id];
|
||||
skip_count--;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
|
||||
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
|
||||
bool is_ep) {
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
int32_t neg_count = 0;
|
||||
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
// Padding and nonlocal routes are both inactive, independently of EP.
|
||||
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
|
||||
for (int e = tid; e <= num_experts; e += stride) {
|
||||
int32_t offset;
|
||||
|
||||
@@ -60,6 +60,16 @@ register_kernel(
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.shuffle_weighted_sum",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def moe_align_block_size(
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
|
||||
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
|
||||
use_int4_w4a16: tl.constexpr,
|
||||
use_int8_w8a16: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
|
||||
token_mask = offs_token < num_valid_tokens
|
||||
|
||||
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
|
||||
if filter_expert and off_experts == -1:
|
||||
# -----------------------------------------------------------
|
||||
# Write back zeros to the output when the expert is not
|
||||
# in the current expert parallel rank.
|
||||
if off_experts == -1:
|
||||
# Padding and nonlocal routes both use -1, including without EP.
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -377,12 +374,10 @@ def fused_moe_kernel(
|
||||
per_channel_quant: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
c_sorted: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
swap_ab: tl.constexpr,
|
||||
FUSE_ADD_TO_OUTPUT: tl.constexpr,
|
||||
MASK_OUTPUT: tl.constexpr,
|
||||
FUSE_SUM_ALL_REDUCE: tl.constexpr,
|
||||
LORA_PRESERVE_BASE: tl.constexpr,
|
||||
ROUTER_TOPK: tl.constexpr,
|
||||
FUSE_SWIGLU: tl.constexpr = False,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
@@ -450,16 +445,10 @@ def fused_moe_kernel(
|
||||
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
|
||||
off_experts = off_experts_i32.to(tl.int64)
|
||||
|
||||
if filter_expert and off_experts == -1:
|
||||
if FUSE_SWIGLU:
|
||||
# C is the half-width post-activation buffer here. Rows owned by a
|
||||
# filtered expert are never read (the down-GEMM CTA for this block
|
||||
# early-exits before loading A), and an N-wide zero store would run
|
||||
# past the row end into a neighboring token's data.
|
||||
return
|
||||
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
|
||||
# Write zeros only when this kernel owns the full output; the experimental LoRA
|
||||
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
|
||||
if off_experts == -1:
|
||||
# The down GEMM never reads inactive sorted or half-width intermediates.
|
||||
# Add/reduce outputs already hold their initial value and must stay intact.
|
||||
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
|
||||
lora_preserve_base: bool = False,
|
||||
fuse_swiglu: bool = False,
|
||||
) -> None:
|
||||
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
|
||||
assert topk_weights.stride(1) == 1
|
||||
assert sorted_token_ids.stride(0) == 1
|
||||
|
||||
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
|
||||
use_int4_w4a16=use_int4_w4a16,
|
||||
use_int8_w8a16=use_int8_w8a16,
|
||||
even_Ks=even_Ks,
|
||||
filter_expert=filter_expert,
|
||||
**config,
|
||||
)
|
||||
|
||||
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
|
||||
per_channel_quant=per_channel_quant,
|
||||
even_Ks=even_Ks,
|
||||
c_sorted=c_sorted,
|
||||
filter_expert=filter_expert,
|
||||
swap_ab=swap_ab,
|
||||
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
|
||||
MASK_OUTPUT=mask_output,
|
||||
LORA_PRESERVE_BASE=lora_preserve_base,
|
||||
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
|
||||
ROUTER_TOPK=router_topk,
|
||||
FUSE_SWIGLU=fuse_swiglu,
|
||||
|
||||
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
|
||||
num_tokens,
|
||||
stride_m,
|
||||
has_expert_map: tl.constexpr,
|
||||
is_ep: tl.constexpr,
|
||||
has_topk_ids: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
size: tl.constexpr,
|
||||
routed_scaling_factor: tl.constexpr,
|
||||
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
|
||||
if routed_scaling_factor != 1.0:
|
||||
b_val = b_val * routed_scaling_factor
|
||||
if has_expert_map:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
|
||||
mapped_id = tl.load(
|
||||
expert_map_ptr + tl.maximum(id_val, 0),
|
||||
mask=m_mask & (id_val >= 0),
|
||||
other=-1,
|
||||
)
|
||||
expert_mask = mapped_id >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
elif is_ep:
|
||||
elif has_topk_ids:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = id_val >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
|
||||
Shape: (num_tokens, top_k).
|
||||
outputs: Optional pre-allocated output tensor.
|
||||
Shape: (num_tokens, hidden_size).
|
||||
topk_ids: Optional indices of the top-k experts. Used when
|
||||
`expert_map` is provided. Shape: (num_tokens, top_k).
|
||||
topk_ids: Optional indices of the top-k experts. Negative ids always
|
||||
skip the corresponding input load. Shape: (num_tokens, top_k).
|
||||
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
||||
indicates an invalid token/expert pair that will be skipped.
|
||||
|
||||
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
|
||||
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
|
||||
# Keep is_ep as a compatible host argument; route validity follows ids.
|
||||
num_tokens, top_k, size = inputs.shape
|
||||
output_shape = (num_tokens, size)
|
||||
if outputs is None:
|
||||
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
|
||||
num_tokens,
|
||||
top_k * size,
|
||||
expert_map is not None,
|
||||
is_ep,
|
||||
topk_ids is not None,
|
||||
top_k,
|
||||
size,
|
||||
1.0 if routed_scaling_factor is None else routed_scaling_factor,
|
||||
|
||||
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Project routes, preserving inactive rows in a caller-provided output.
|
||||
|
||||
Inactive rows are zero when this function allocates the output.
|
||||
"""
|
||||
device = a.device
|
||||
|
||||
# Allocate output if not provided
|
||||
if c_or_none is not None:
|
||||
c = c_or_none
|
||||
else:
|
||||
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
|
||||
# Early return for zero-size M
|
||||
if size_m == 0:
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Reduce routed rows while skipping inactive permutation entries."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
|
||||
def _shuffle_weighted_sum_kernel(
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
width: tl.constexpr,
|
||||
rows,
|
||||
topk: tl.constexpr,
|
||||
input_stride: tl.constexpr,
|
||||
output_stride: tl.constexpr,
|
||||
weight_stride0,
|
||||
weight_stride1,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
total = tl.full((BLOCK,), 0, tl.float32)
|
||||
for slot in range(topk):
|
||||
row = tl.load(permutation + token * topk + slot).to(tl.int64)
|
||||
active = (row >= 0) & (row < rows)
|
||||
values = tl.load(
|
||||
inputs + row * input_stride + columns,
|
||||
mask=active & (columns < width),
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
factor = tl.load(
|
||||
weights + token * weight_stride0 + slot * weight_stride1,
|
||||
mask=active,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
total += values * factor
|
||||
tl.store(
|
||||
output + token.to(tl.int64) * output_stride + columns,
|
||||
total,
|
||||
mask=columns < width,
|
||||
)
|
||||
|
||||
|
||||
def shuffle_weighted_sum(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
permutation: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
) -> None:
|
||||
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
|
||||
assert inputs.ndim == output.ndim == weights.ndim == 2
|
||||
assert inputs.shape[1] == output.shape[1]
|
||||
assert inputs.stride(1) == output.stride(1) == 1
|
||||
assert weights.shape[0] == output.shape[0]
|
||||
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
|
||||
if output.numel() == 0:
|
||||
return
|
||||
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
output.shape[1],
|
||||
inputs.shape[0],
|
||||
weights.shape[1],
|
||||
inputs.stride(0),
|
||||
output.stride(0),
|
||||
weights.stride(0),
|
||||
weights.stride(1),
|
||||
BLOCK=256,
|
||||
)
|
||||
@@ -125,6 +125,10 @@ class _StageExecutor:
|
||||
|
||||
# handling DP attention
|
||||
forward_batch: ForwardBatch = inputs["forward_batch"]
|
||||
self._child_ctx = replace(
|
||||
child_ctx if child_ctx is not None else get_forward_context(),
|
||||
moe_num_token_non_padded=forward_batch.moe_num_token_non_padded,
|
||||
)
|
||||
self._global_dp_buffer_len = forward_batch.global_dp_buffer_len
|
||||
self._local_dp_buffer_len = forward_batch.tbo_padded_len
|
||||
self._global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
|
||||
@@ -39,13 +39,16 @@ class ExpertLocationDispatchInfo:
|
||||
rank_invariant: bool = False
|
||||
|
||||
@classmethod
|
||||
def init_new(cls, layer_id: int):
|
||||
def init_new(cls, layer_id: int, *, is_draft: bool = False):
|
||||
if is_draft:
|
||||
return None
|
||||
ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm
|
||||
expert_location_metadata = get_global_expert_location_metadata()
|
||||
assert expert_location_metadata is not None
|
||||
|
||||
if ep_dispatch_algorithm is None:
|
||||
return None
|
||||
expert_location_metadata = get_global_expert_location_metadata()
|
||||
assert expert_location_metadata is not None, (
|
||||
"Expert dispatch requires placement metadata"
|
||||
)
|
||||
|
||||
return cls(
|
||||
ep_dispatch_algorithm=ep_dispatch_algorithm,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutput,
|
||||
_zero_topk_weights_padded_region,
|
||||
capture_routed_experts_if_allowed,
|
||||
select_experts,
|
||||
)
|
||||
@@ -59,6 +59,8 @@ def fused_topk_npu(
|
||||
)
|
||||
|
||||
if renormalize:
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
topk_weights = l1_norm(
|
||||
topk_weights
|
||||
if topk_config.num_shared_topk_slots == 0
|
||||
@@ -109,9 +111,44 @@ def fused_topk_npu(
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
|
||||
if expert_location_dispatch_info is not None:
|
||||
topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info)
|
||||
get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
|
||||
return _finalize_topk_npu(
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
router_logits=router_logits,
|
||||
topk_config=topk_config,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
layer_id=layer_id,
|
||||
)
|
||||
|
||||
|
||||
def _finalize_topk_npu(
|
||||
*,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
router_logits,
|
||||
topk_config,
|
||||
num_token_non_padded,
|
||||
expert_location_dispatch_info,
|
||||
layer_id,
|
||||
):
|
||||
num_shared = topk_config.num_shared_topk_slots
|
||||
routed_ids = topk_ids[:, :-num_shared] if num_shared else topk_ids
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, routed_ids)
|
||||
routed_ids = topk_ids_logical_to_physical(routed_ids, expert_location_dispatch_info)
|
||||
topk_ids = (
|
||||
torch.cat([routed_ids, topk_ids[:, -num_shared:]], dim=-1)
|
||||
if num_shared
|
||||
else routed_ids
|
||||
)
|
||||
# NPU dispatch retains valid IDs; duplicating a padded row's routes onto
|
||||
# expert zero can exceed per-expert capacity in fused dispatch backends.
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
recorder_ids = routed_ids
|
||||
if num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_ids = routed_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
get_global_expert_distribution_recorder().on_select_experts(topk_ids=recorder_ids)
|
||||
return StandardTopKOutput(topk_weights, topk_ids, router_logits)
|
||||
|
||||
@@ -1209,10 +1209,6 @@ class AiterAttnBackend(AttentionBackend):
|
||||
bs, forward_batch.seq_lens, forward_batch.req_pool_indices, max_kv_len
|
||||
)
|
||||
|
||||
# Build cu_seqlens_q from this batch's extend lengths. The standard
|
||||
# prefill metadata path leaves self.qo_indptr unset (qo_indptr=None in
|
||||
# ForwardMetadata), so relying on it corrupts multi-sequence batches
|
||||
# (only bs=1 happens to work).
|
||||
cu_seqlens_q = torch.zeros(bs + 1, dtype=torch.int32, device=q.device)
|
||||
cu_seqlens_q[1:] = torch.cumsum(
|
||||
forward_batch.extend_seq_lens.to(torch.int32), dim=0
|
||||
@@ -2598,7 +2594,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
self.indices_updater_prefill.kv_indptr,
|
||||
self.indices_updater_prefill.kv_indices,
|
||||
None,
|
||||
self.indices_updater_prefill.qo_indptr[: bs + 1],
|
||||
None,
|
||||
max(forward_batch.extend_seq_lens_cpu),
|
||||
forward_batch.seq_lens_cpu.max().item(),
|
||||
@@ -4423,7 +4419,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.k_scale.reshape(1), # Q is cast at unit scale.
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
@@ -4499,7 +4495,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fp8_q_descale.reshape(1),
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
@@ -4538,8 +4534,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fp8_q_descale.reshape(1),
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
self.forward_metadata.max_q_len,
|
||||
self.forward_metadata.max_q_len,
|
||||
softmax_scale=layer.scaling,
|
||||
@@ -4628,7 +4624,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_cache,
|
||||
v_cache,
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_arg,
|
||||
page_table,
|
||||
self.forward_metadata.max_q_len,
|
||||
|
||||
@@ -94,13 +94,13 @@ def forward_extend_vectorized_5d(
|
||||
kv_indices_lin = torch.arange(
|
||||
total_tokens, dtype=torch.int32, device=k_lin.device
|
||||
)
|
||||
kv_indptr_lin = backend.qo_indptr[:bs0]
|
||||
kv_indptr_lin = backend.forward_metadata.qo_indptr[:bs0]
|
||||
max_q = int(backend.forward_metadata.max_q_len)
|
||||
o = mha_batch_prefill_func(
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_lin,
|
||||
v_lin,
|
||||
backend.qo_indptr[:bs0],
|
||||
backend.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_lin,
|
||||
kv_indices_lin,
|
||||
max_q,
|
||||
@@ -187,7 +187,7 @@ def forward_extend_vectorized_5d(
|
||||
q_local.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_lin,
|
||||
v_lin,
|
||||
backend.qo_indptr[:bs0],
|
||||
backend.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_lin,
|
||||
kv_indices_lin,
|
||||
max_q,
|
||||
|
||||
@@ -10,7 +10,6 @@ from sglang.srt.utils import is_cuda
|
||||
_is_cuda = is_cuda()
|
||||
if _is_cuda:
|
||||
from sgl_kernel import (
|
||||
apply_shuffle_mul_sum,
|
||||
es_fp8_blockwise_scaled_grouped_mm,
|
||||
es_sm100_mxfp8_blockscaled_grouped_mm,
|
||||
es_sm100_mxfp8_blockscaled_grouped_quant,
|
||||
@@ -23,6 +22,7 @@ if _is_cuda:
|
||||
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
|
||||
shuffle_rows_with_scales,
|
||||
)
|
||||
from sglang.kernels.ops.moe.shuffle_weighted_sum import shuffle_weighted_sum
|
||||
|
||||
|
||||
def cutlass_fused_experts_fp8(
|
||||
@@ -142,8 +142,9 @@ def cutlass_fused_experts_fp8(
|
||||
topk = topk_ids.size(1)
|
||||
device = a.device
|
||||
|
||||
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
# prepare_moe_input writes only live routes; gathers still span the tail.
|
||||
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
|
||||
|
||||
if use_mxfp8:
|
||||
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
|
||||
@@ -345,5 +346,5 @@ def cutlass_fused_experts_fp8(
|
||||
if output is None:
|
||||
output = torch.empty((m, k), device=device, dtype=out_dtype)
|
||||
|
||||
apply_shuffle_mul_sum(c2, output, c_map, topk_weights.to(out_dtype))
|
||||
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
|
||||
return output
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda, is_cuda_alike
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -128,8 +127,8 @@ def cutlass_w4a8_moe(
|
||||
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
|
||||
|
||||
device = a.device
|
||||
if get_parallel().moe_ep_size > 1:
|
||||
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
|
||||
# Both graph padding and nonlocal routes use -1, including without EP.
|
||||
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
|
||||
|
||||
src2dst = cutlass_w4_run_moe_ep_preproess(
|
||||
topk_ids,
|
||||
@@ -144,7 +143,11 @@ def cutlass_w4a8_moe(
|
||||
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
|
||||
if a1_scale is None:
|
||||
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
|
||||
per_tensor_absmax_fp8(a, a1_scale)
|
||||
# Inactive rows must not change the quantization of live expert inputs.
|
||||
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
|
||||
scale_input = a.masked_fill(inactive_rows, 0.0)
|
||||
per_tensor_absmax_fp8(scale_input, a1_scale)
|
||||
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
|
||||
|
||||
pre_reorder_for_cutlass_moe(
|
||||
a,
|
||||
|
||||
@@ -305,15 +305,14 @@ def fused_marlin_moe(
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
# Marlin skips masked expert rows, so their shared cache must start at zero.
|
||||
intermediate_cache13 = torch.zeros(
|
||||
(M * topk_ids.shape[1] * max(gemm1_n, K),),
|
||||
intermediate_cache13 = torch.empty(
|
||||
(M * topk * max(gemm1_n, K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * gemm1_n]
|
||||
intermediate_cache1 = intermediate_cache1.view(-1, gemm1_n)
|
||||
intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K]
|
||||
intermediate_cache3 = intermediate_cache13[: M * topk * K]
|
||||
intermediate_cache3 = intermediate_cache3.view(-1, K)
|
||||
|
||||
use_atomic_add = (
|
||||
@@ -381,9 +380,9 @@ def fused_marlin_moe(
|
||||
else:
|
||||
raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}")
|
||||
|
||||
if expert_map is not None:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
# Gate-up is dead after activation, so its storage can hold the down output.
|
||||
# Skipped down routes must start at zero before the top-k reduction.
|
||||
intermediate_cache3.zero_()
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
intermediate_cache3,
|
||||
|
||||
@@ -74,6 +74,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_global_dwdp_manager,
|
||||
get_parallel,
|
||||
get_server_args,
|
||||
@@ -367,7 +368,8 @@ class FusedMoE(torch.nn.Module):
|
||||
self.params_dtype = params_dtype
|
||||
self.layer_name = prefix
|
||||
self.layer_id = layer_id
|
||||
self.uses_expert_location = uses_expert_location
|
||||
self.is_draft = get_flags().moe.in_speculative_scope
|
||||
self.uses_expert_location = uses_expert_location and not self.is_draft
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.num_experts = num_experts
|
||||
@@ -1559,6 +1561,14 @@ class FusedMoE(torch.nn.Module):
|
||||
|
||||
return forward_fuseep(self, hidden_states, topk_output)
|
||||
if is_in_tc_piecewise_cuda_graph():
|
||||
if (
|
||||
TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
and topk_output.expert_location_dispatch_info is not None
|
||||
):
|
||||
# The opaque logits-only op cannot carry placement metadata.
|
||||
return self.forward_impl(
|
||||
hidden_states, topk_output, pre_quant_input=pre_quant_input
|
||||
)
|
||||
if TopKOutputChecker.format_is_standard(topk_output):
|
||||
return moe_forward_piecewise_cuda_graph_impl(
|
||||
hidden_states,
|
||||
@@ -1872,7 +1882,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
|
||||
) -> torch.Tensor:
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=TopKConfig(
|
||||
top_k=top_k,
|
||||
topk_group=topk_group,
|
||||
@@ -1880,6 +1890,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
|
||||
correction_bias=correction_bias,
|
||||
renormalize=renormalize,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture,
|
||||
layer_id=layer_id,
|
||||
),
|
||||
)
|
||||
forward_context = get_tc_piecewise_forward_context()
|
||||
|
||||
@@ -21,11 +21,10 @@ from sglang.srt.layers.moe.topk import (
|
||||
_mask_topk_ids_padded_region,
|
||||
_zero_topk_weights_padded_region,
|
||||
remap_topk_for_per_rank_shared_slots,
|
||||
resolve_routing_inputs,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
|
||||
from sglang.srt.runtime_context import get_flags, get_model
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -46,9 +45,13 @@ class HashTopK(nn.Module):
|
||||
routed_scaling_factor=1.5,
|
||||
apply_routed_scaling_factor_on_output=False,
|
||||
layer_id: Optional[int] = None,
|
||||
is_draft: Optional[bool] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
self.is_draft = (
|
||||
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
|
||||
)
|
||||
|
||||
self.shared_experts = get_model().shared_experts
|
||||
self.enable_waterfill = self.shared_experts.use_waterfill
|
||||
@@ -75,6 +78,9 @@ class HashTopK(nn.Module):
|
||||
"with fused shared experts; pass --disable-shared-experts-fusion."
|
||||
)
|
||||
|
||||
def routing_inputs(self):
|
||||
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
|
||||
|
||||
@property
|
||||
def num_shared_topk_slots(self) -> int:
|
||||
return self.shared_experts.num_shared_topk_slots
|
||||
@@ -100,7 +106,8 @@ class HashTopK(nn.Module):
|
||||
self, device: torch.device, *, layer_id: Optional[int] = None
|
||||
):
|
||||
topk = self.topk - self.num_shared_topk_slots
|
||||
if layer_id is not None:
|
||||
if not self.is_draft:
|
||||
layer_id = self.layer_id
|
||||
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
|
||||
|
||||
lplb_solver = get_global_lplb_solver(layer_id)
|
||||
@@ -226,6 +233,12 @@ class HashTopK(nn.Module):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
):
|
||||
num_token_non_padded, expert_location_dispatch_info = resolve_routing_inputs(
|
||||
layer_id=self.layer_id,
|
||||
is_draft=self.is_draft,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
assert input_ids.shape[0] == hidden_states.shape[0] == router_logits.shape[0], (
|
||||
f"{input_ids.shape=} {hidden_states.shape=} {router_logits.shape=}"
|
||||
)
|
||||
@@ -252,6 +265,9 @@ class HashTopK(nn.Module):
|
||||
topk_weights = topk_weights * self.routed_scaling_factor
|
||||
|
||||
num_shared_topk_slots = self.num_shared_topk_slots
|
||||
routed_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots] if num_shared_topk_slots else topk_ids
|
||||
)
|
||||
log2phy_prob = None
|
||||
if (
|
||||
expert_location_dispatch_info is not None
|
||||
@@ -263,18 +279,13 @@ class HashTopK(nn.Module):
|
||||
|
||||
lplb_solver = get_global_lplb_solver(self.layer_id)
|
||||
if lplb_solver is not None:
|
||||
log2phy_prob = lplb_solver.solve(topk_ids)
|
||||
log2phy_prob = lplb_solver.solve(routed_ids)
|
||||
|
||||
recorder_topk_ids = None
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
routed_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
recorder_topk_ids = routed_cols if num_shared_topk_slots else None
|
||||
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
|
||||
# The trailing columns are placeholders; the remap appends the real ones.
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
recorder_topk_ids = routed_cols
|
||||
|
||||
num_physical_routed_experts = (
|
||||
expert_location_dispatch_info.num_physical_experts
|
||||
if expert_location_dispatch_info is not None
|
||||
@@ -292,11 +303,13 @@ class HashTopK(nn.Module):
|
||||
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
elif num_shared_topk_slots:
|
||||
topk_ids = torch.cat(
|
||||
[routed_cols, topk_ids[:, -num_shared_topk_slots:]], dim=-1
|
||||
)
|
||||
if is_hip():
|
||||
else:
|
||||
topk_ids = routed_cols
|
||||
if _is_hip or _is_npu:
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
else:
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
@@ -304,6 +317,11 @@ class HashTopK(nn.Module):
|
||||
_mask_topk_ids_padded_region(recorder_topk_ids, num_token_non_padded)
|
||||
if recorder_topk_ids is None:
|
||||
recorder_topk_ids = topk_ids
|
||||
if _is_npu and num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_topk_ids = recorder_topk_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_topk_ids
|
||||
)
|
||||
@@ -311,7 +329,7 @@ class HashTopK(nn.Module):
|
||||
topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits
|
||||
)
|
||||
topk_output = self._apply_waterfill(topk_output, hidden_states.shape[0])
|
||||
if is_hip():
|
||||
if _is_hip or _is_npu:
|
||||
_zero_topk_weights_padded_region(
|
||||
topk_output.topk_weights, num_token_non_padded
|
||||
)
|
||||
|
||||
@@ -23,7 +23,6 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.moe.mega_moe_sm90 import (
|
||||
@@ -263,16 +262,6 @@ def _run_mega_routed(
|
||||
if num_tokens > 0:
|
||||
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
|
||||
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
|
||||
num_token_non_padded = (
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
)
|
||||
if (
|
||||
not moe.is_hash
|
||||
and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None
|
||||
@@ -281,15 +270,11 @@ def _run_mega_routed(
|
||||
moe,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
)
|
||||
else:
|
||||
topk_output = moe.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
topk_ids = topk_output.topk_ids
|
||||
|
||||
@@ -14,7 +14,6 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
@@ -307,25 +306,16 @@ def _run_mega_routed(
|
||||
topk_output = moe.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.num_token_non_padded
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
x_in = hidden_states
|
||||
topk_ids = topk_output.topk_ids.to(torch.int32)
|
||||
topk_weights = topk_output.topk_weights.to(torch.float32)
|
||||
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
|
||||
num_token_non_padded, _ = moe.topk.routing_inputs()
|
||||
if num_token_non_padded is not None:
|
||||
_fill_padded_rows(
|
||||
topk_ids,
|
||||
forward_batch.num_token_non_padded,
|
||||
num_token_non_padded,
|
||||
experts,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_model,
|
||||
@@ -110,6 +111,10 @@ from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
@@ -279,6 +284,8 @@ class TopKConfig:
|
||||
# Draft-side MoE blocks set this False so they never write the target's
|
||||
# process-global routed-experts capture buffer.
|
||||
allow_routed_experts_capture: bool = True
|
||||
# Lazy materialization still needs the producer's solver and capture slot.
|
||||
layer_id: Optional[int] = None
|
||||
_correction_bias_dtype_cache: Optional[torch.Tensor] = field(
|
||||
default=None, init=False, repr=False, compare=False
|
||||
)
|
||||
@@ -393,10 +400,10 @@ class TritonKernelTopKOutput(NamedTuple):
|
||||
|
||||
|
||||
class BypassedTopKOutput(NamedTuple):
|
||||
"""Bypassed top-k output format."""
|
||||
"""Lazy routing that can be materialized or consumed by an in-kernel router."""
|
||||
|
||||
hidden_states: torch.Tensor
|
||||
router_logits: torch.Tensor
|
||||
raw_router_logits: torch.Tensor
|
||||
topk_config: TopKConfig
|
||||
num_token_non_padded: Optional[torch.Tensor] = None
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None
|
||||
@@ -405,14 +412,23 @@ class BypassedTopKOutput(NamedTuple):
|
||||
def format(self) -> TopKOutputFormat:
|
||||
return TopKOutputFormat.BYPASSED
|
||||
|
||||
@property
|
||||
def router_logits(self) -> torch.Tensor:
|
||||
if self.expert_location_dispatch_info is not None:
|
||||
raise NotImplementedError(
|
||||
"Expert placement remapping requires materialized TopK routing; "
|
||||
"this MoE kernel routes directly from logits."
|
||||
)
|
||||
return self.raw_router_logits
|
||||
|
||||
def to_standard(self, layer_id: Optional[int] = None) -> StandardTopKOutput:
|
||||
"""Materialize routing tensors. Used by MoE kernels that need explicit
|
||||
topk_ids / topk_weights rather than doing routing internally."""
|
||||
return select_experts(
|
||||
hidden_states=self.hidden_states,
|
||||
router_logits=self.router_logits,
|
||||
router_logits=self.raw_router_logits,
|
||||
topk_config=self.topk_config,
|
||||
layer_id=layer_id,
|
||||
layer_id=self.topk_config.layer_id if layer_id is None else layer_id,
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
expert_location_dispatch_info=self.expert_location_dispatch_info,
|
||||
)
|
||||
@@ -630,6 +646,24 @@ def _can_bypass_trtllm_routing(config: TopKConfig) -> bool:
|
||||
# -------------------------------- TopK ---------------------------------------
|
||||
|
||||
|
||||
def resolve_routing_inputs(
|
||||
*,
|
||||
layer_id: int,
|
||||
is_draft: bool,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
):
|
||||
if num_token_non_padded is None and has_forward_context():
|
||||
get_bound = get_forward_context().moe_num_token_non_padded
|
||||
if get_bound is not None:
|
||||
num_token_non_padded = get_bound()
|
||||
if is_draft:
|
||||
expert_location_dispatch_info = None
|
||||
elif expert_location_dispatch_info is None:
|
||||
expert_location_dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id)
|
||||
return num_token_non_padded, expert_location_dispatch_info
|
||||
|
||||
|
||||
class TopK(BaseFusedOp):
|
||||
"""
|
||||
Parameters:
|
||||
@@ -649,6 +683,7 @@ class TopK(BaseFusedOp):
|
||||
top_k: int,
|
||||
*,
|
||||
layer_id: int,
|
||||
is_draft: Optional[bool] = None,
|
||||
use_grouped_topk: bool = False,
|
||||
topk_group: Optional[int] = None,
|
||||
num_expert_group: Optional[int] = None,
|
||||
@@ -674,6 +709,9 @@ class TopK(BaseFusedOp):
|
||||
assert num_expert_group is not None and topk_group is not None
|
||||
|
||||
self.layer_id = layer_id
|
||||
self.is_draft = (
|
||||
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
|
||||
)
|
||||
self._runner_backend_bound = False
|
||||
|
||||
shared_experts = get_model().shared_experts
|
||||
@@ -686,6 +724,7 @@ class TopK(BaseFusedOp):
|
||||
self.is_fp4_experts = is_fp4_experts
|
||||
self.topk_config = TopKConfig(
|
||||
top_k=top_k + num_shared_slots,
|
||||
layer_id=layer_id,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
@@ -703,7 +742,21 @@ class TopK(BaseFusedOp):
|
||||
scoring_func=scoring_func,
|
||||
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
|
||||
fused_gate_packed_ids=fused_gate_packed_ids,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture
|
||||
and not self.is_draft,
|
||||
)
|
||||
|
||||
def routing_inputs(self):
|
||||
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
|
||||
|
||||
def _resolve_routing_inputs(
|
||||
self, num_token_non_padded, expert_location_dispatch_info
|
||||
):
|
||||
return resolve_routing_inputs(
|
||||
layer_id=self.layer_id,
|
||||
is_draft=self.is_draft,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
|
||||
def bind_runner_backend(
|
||||
@@ -778,6 +831,11 @@ class TopK(BaseFusedOp):
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if router_logits_partials is not None:
|
||||
rocm_router_reduce_partials(router_logits_partials, router_logits)
|
||||
self.topk_config.torch_native = True
|
||||
@@ -805,6 +863,12 @@ class TopK(BaseFusedOp):
|
||||
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
|
||||
fixed-order sum is the logits; router_logits is then a buffer the fused gate
|
||||
fills, and any other reader first reduces the partials into it."""
|
||||
explicit_dispatch_info = expert_location_dispatch_info is not None
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if dynamic_expert_bias is not None:
|
||||
output_format = TopKOutputFormat.STANDARD
|
||||
elif self.topk_config.output_format is not None:
|
||||
@@ -814,6 +878,17 @@ class TopK(BaseFusedOp):
|
||||
"TopK requires the owning experts' runner backend or an explicit output format."
|
||||
)
|
||||
|
||||
requires_remapping = expert_location_dispatch_info is not None and (
|
||||
explicit_dispatch_info
|
||||
or is_expert_remapping_enabled()
|
||||
or expert_location_dispatch_info.ep_dispatch_algorithm in ("fake", "lp")
|
||||
)
|
||||
if output_format == TopKOutputFormat.TRITON_KERNEL and requires_remapping:
|
||||
raise NotImplementedError(
|
||||
"Expert placement remapping requires a MoE runner with standard TopK routing; "
|
||||
f"got {output_format.name}."
|
||||
)
|
||||
|
||||
if (
|
||||
router_logits_partials is not None
|
||||
and output_format != TopKOutputFormat.STANDARD
|
||||
@@ -833,6 +908,7 @@ class TopK(BaseFusedOp):
|
||||
router_logits,
|
||||
self.topk_config.top_k,
|
||||
sm_first=not self.topk_config.renormalize,
|
||||
n_rows=num_token_non_padded,
|
||||
)
|
||||
return TritonKernelTopKOutput(
|
||||
a_ragged_metadata,
|
||||
@@ -844,10 +920,12 @@ class TopK(BaseFusedOp):
|
||||
elif output_format == TopKOutputFormat.BYPASSED:
|
||||
return BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=self.topk_config,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
expert_location_dispatch_info=(
|
||||
expert_location_dispatch_info if requires_remapping else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.topk_config.torch_native = False
|
||||
@@ -875,6 +953,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
topk_output = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
layer_id=self.layer_id,
|
||||
@@ -895,6 +978,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if dynamic_expert_bias is not None:
|
||||
self.topk_config.torch_native = False
|
||||
return select_experts(
|
||||
@@ -923,12 +1011,13 @@ class TopK(BaseFusedOp):
|
||||
) -> TopKOutput:
|
||||
"""Return an empty topk output for a rank with zero tokens this forward.
|
||||
|
||||
When ``layer_id`` is provided and the active dispatch algorithm is LP,
|
||||
also calls ``LPLBSolver.solve(empty)`` so that this rank participates
|
||||
in the EP all-reduce. Without this, an empty rank would skip the
|
||||
collective and deadlock under DP-attention.
|
||||
Target layers call their owning ``LPLBSolver.solve(empty)`` so this
|
||||
rank participates in the EP all-reduce even without tokens. Draft
|
||||
layers skip the target solver. The optional ``layer_id`` is retained
|
||||
for caller compatibility; the router's owning layer determines the solver.
|
||||
"""
|
||||
if layer_id is not None:
|
||||
if not self.is_draft:
|
||||
layer_id = self.layer_id
|
||||
# Skip the full ExpertLocationDispatchInfo allocation — we only
|
||||
# need the per-layer solver to participate in the EP all-reduce.
|
||||
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
|
||||
@@ -972,6 +1061,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
self.topk_config.torch_native = True
|
||||
# [NOTE] XPU device support for topk kernels
|
||||
# - support 'topk_softmax' and 'topk_sigmoid'
|
||||
@@ -1816,6 +1910,12 @@ def _mask_topk_ids_padded_region(
|
||||
) -> None:
|
||||
if num_token_non_padded is None:
|
||||
return
|
||||
if not topk_ids.is_cuda:
|
||||
if _is_npu:
|
||||
return
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
topk_ids.masked_fill_((rows >= num_token_non_padded)[:, None], fill_value)
|
||||
return
|
||||
# TODO: let the kernel support other dtypes
|
||||
if _is_cuda and topk_ids.dtype == torch.int32 and fill_value == -1:
|
||||
mask_topk_ids(topk_ids, num_token_non_padded)
|
||||
@@ -1834,6 +1934,10 @@ def _zero_topk_weights_padded_region(
|
||||
):
|
||||
if num_token_non_padded is None:
|
||||
return
|
||||
if not topk_weights.is_cuda:
|
||||
rows = torch.arange(topk_weights.shape[0], device=topk_weights.device)
|
||||
topk_weights.masked_fill_((rows >= num_token_non_padded)[:, None], 0.0)
|
||||
return
|
||||
if _can_fuse_padded_region(topk_weights):
|
||||
_fill_padded_rows(topk_weights, num_token_non_padded, 0.0)
|
||||
return
|
||||
@@ -2420,12 +2524,27 @@ def _post_process_topk_ids(
|
||||
|
||||
lplb_solver = get_global_lplb_solver(layer_id)
|
||||
if lplb_solver is not None:
|
||||
log2phy_prob = lplb_solver.solve(topk_ids)
|
||||
lp_routed_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots]
|
||||
if router_wrote_shared_cols
|
||||
else topk_ids
|
||||
)
|
||||
log2phy_prob = lplb_solver.solve(lp_routed_ids)
|
||||
|
||||
if log2phy_prob is not None:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
if router_wrote_shared_cols:
|
||||
shared_cols = topk_ids[:, -num_shared_topk_slots:]
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = topk_ids[:, :-num_shared_topk_slots]
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif router_wrote_shared_cols:
|
||||
# Shared IDs are outside EPLB's routed-expert table for both global
|
||||
@@ -2506,6 +2625,21 @@ def _post_process_topk_ids(
|
||||
# That final pass re-zeros after any shared-expert append/remap, so a
|
||||
# second zeroing here would be redundant (zeroing is idempotent).
|
||||
|
||||
else:
|
||||
if num_shared_topk_slots > 0:
|
||||
shared_cols = topk_ids[:, -num_shared_topk_slots:]
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info
|
||||
)
|
||||
_mask_topk_ids_padded_region(routed_cols, num_token_non_padded)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = routed_cols
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
|
||||
if recorder_topk_ids is None and not recorder_was_fused:
|
||||
recorder_topk_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
|
||||
@@ -2598,6 +2732,14 @@ def _post_process_topk_ids(
|
||||
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
|
||||
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
|
||||
|
||||
if _is_npu:
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
if recorder_topk_ids is not None and num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_topk_ids = recorder_topk_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
|
||||
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
|
||||
# Shared-expert append/remap can introduce non-zero weights after the
|
||||
# initial HIP padding mask above. Ensure padded tokens leave this helper
|
||||
@@ -2750,10 +2892,6 @@ def select_experts(
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
)
|
||||
elif torch_native and custom_routing_function is None:
|
||||
assert num_token_non_padded is None, (
|
||||
"num_token_non_padded is not yet supported in fused_topk_native"
|
||||
)
|
||||
assert expert_location_dispatch_info is None
|
||||
fold_scale_after_routing = True
|
||||
topk_weights, topk_ids = fused_topk_native(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -387,6 +387,9 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
_map_topk_ids_to_local_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
|
||||
|
||||
hidden_states = hidden_states.contiguous()
|
||||
@@ -398,7 +401,7 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
|
||||
local_topk_ids = topk_ids.to(torch.int32)
|
||||
if expert_map is not None:
|
||||
local_topk_ids = expert_map[local_topk_ids]
|
||||
local_topk_ids = _map_topk_ids_to_local_experts(local_topk_ids, expert_map)
|
||||
local_topk_ids = local_topk_ids.contiguous()
|
||||
topk_weights = topk_scores.to(torch.float32).contiguous()
|
||||
|
||||
@@ -430,13 +433,13 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache13 = torch.empty(
|
||||
(num_tokens * topk * max(gemm1_n, size_k),),
|
||||
cache13 = torch.zeros(
|
||||
(num_tokens * topk * (gemm1_n + size_k),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
|
||||
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
|
||||
cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k)
|
||||
|
||||
cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
@@ -525,6 +528,7 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
|
||||
hidden_states, topk_scores, topk_ids
|
||||
)
|
||||
topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0)
|
||||
if self._post_expert_norm is not post_expert_norm:
|
||||
self.set_post_expert_norm(post_expert_norm)
|
||||
if self._post_expert_norm_weight is not None:
|
||||
|
||||
@@ -18,6 +18,9 @@ from sglang.srt.hardware_backend.npu.quantization.moe_methods import (
|
||||
from sglang.srt.hardware_backend.npu.utils import npu_format_cast
|
||||
from sglang.srt.layers.linear import LinearBase
|
||||
from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend, get_moe_runner_backend
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
FusedMoEMethodBase,
|
||||
@@ -41,24 +44,22 @@ _is_musa = is_musa()
|
||||
_is_npu = is_npu()
|
||||
|
||||
if _is_cuda:
|
||||
from sgl_kernel import moe_align_block_size, moe_sum
|
||||
from sgl_kernel import moe_sum
|
||||
from sgl_kernel.quantization import (
|
||||
ggml_dequantize,
|
||||
ggml_moe_a8,
|
||||
ggml_moe_a8_vec,
|
||||
ggml_moe_get_block_size,
|
||||
ggml_mul_mat_a8,
|
||||
ggml_mul_mat_vec_a8,
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
|
||||
elif _is_musa:
|
||||
from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul
|
||||
from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul
|
||||
from sgl_kernel.quantization import (
|
||||
ggml_dequantize,
|
||||
ggml_moe_a8,
|
||||
ggml_moe_a8_vec,
|
||||
ggml_moe_get_block_size,
|
||||
ggml_mul_mat_a8,
|
||||
ggml_mul_mat_vec_a8,
|
||||
)
|
||||
@@ -180,6 +181,16 @@ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
|
||||
MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
|
||||
|
||||
|
||||
def ggml_moe_get_block_size(qweight_type: int) -> int:
|
||||
# MOE_X in the CUDA/MUSA GGUF MMQ kernels is 4. The released wheel's
|
||||
# Tensor-free metadata operator has no catch-all dispatch registration.
|
||||
return (
|
||||
(8 if _is_hip else 4)
|
||||
if qweight_type in MMQ_QUANT_TYPES and qweight_type != WeightType.Q8_1
|
||||
else 0
|
||||
)
|
||||
|
||||
|
||||
def dequantize_gguf_weight(
|
||||
qweight: torch.Tensor, qweight_type: int, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
@@ -239,6 +250,8 @@ def fused_moe_gguf(
|
||||
return gelu_and_mul(x)
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
inactive = topk_ids < 0
|
||||
x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0)
|
||||
out_hidden_states = torch.empty_like(x)
|
||||
# unless we decent expert reuse we are better off running moe_vec kernel
|
||||
if (
|
||||
@@ -254,6 +267,11 @@ def fused_moe_gguf(
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, BLOCK_SIZE, E
|
||||
)
|
||||
# Released MMQ kernels inspect one slack block before their bound check.
|
||||
block_starts = (
|
||||
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
|
||||
)
|
||||
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
|
||||
out = ggml_moe_a8(
|
||||
x,
|
||||
w1,
|
||||
@@ -265,6 +283,7 @@ def fused_moe_gguf(
|
||||
top_k,
|
||||
num_tokens,
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
out = act(out)
|
||||
out = ggml_moe_a8(
|
||||
out,
|
||||
@@ -277,8 +296,10 @@ def fused_moe_gguf(
|
||||
1,
|
||||
num_tokens * top_k,
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
valid_weights = topk_weights.masked_fill(inactive, 0.0)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
valid_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
# TODO(FlamingoPg): maybe we can use moe_sum_reduce here?
|
||||
moe_sum(out, out_hidden_states)
|
||||
@@ -287,14 +308,20 @@ def fused_moe_gguf(
|
||||
E, N, _ = w1.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
out = ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
|
||||
# Older vector kernels form weight addresses before checking -1.
|
||||
# They have no per-expert capacity buffer; discard their dummy slots.
|
||||
safe_ids = topk_ids.clamp_min(0)
|
||||
out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
out = act(out)
|
||||
|
||||
out = ggml_moe_a8_vec(
|
||||
out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
|
||||
out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
valid_weights = topk_weights.masked_fill(inactive, 0.0)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
valid_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
moe_sum(out, out_hidden_states)
|
||||
else:
|
||||
@@ -305,8 +332,10 @@ def fused_moe_gguf(
|
||||
)
|
||||
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
|
||||
inp = x[tok].reshape((1,) + x.shape[1:])
|
||||
current_hidden_state = None
|
||||
current_hidden_state = torch.zeros_like(inp)
|
||||
for ww, ii in zip(w, idx):
|
||||
if ii < 0:
|
||||
continue
|
||||
expert_up = w1[ii]
|
||||
|
||||
out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
|
||||
@@ -316,10 +345,7 @@ def fused_moe_gguf(
|
||||
current_state = fused_mul_mat_gguf(
|
||||
out, expert_down, qweight_type2
|
||||
).mul_(ww)
|
||||
if current_hidden_state is None:
|
||||
current_hidden_state = current_state
|
||||
else:
|
||||
current_hidden_state.add_(current_state)
|
||||
current_hidden_state.add_(current_state)
|
||||
out_hidden_states[tok] = current_hidden_state
|
||||
return out_hidden_states
|
||||
|
||||
@@ -565,8 +591,6 @@ class GGUFMoEMethod(FusedMoEMethodBase):
|
||||
layer: torch.nn.Module,
|
||||
dispatch_output: StandardDispatchOutput,
|
||||
) -> CombineInput:
|
||||
assert self.fused_experts is None
|
||||
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
|
||||
assert self.moe_runner_config.activation == "silu", (
|
||||
|
||||
@@ -71,8 +71,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
|
||||
assert runner_config.activation == "silu", "Only SiLU activation is supported."
|
||||
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
|
||||
@@ -119,9 +119,13 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
)
|
||||
|
||||
# Stage 1: Gate/Up (Marlin)
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -165,11 +169,6 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: Down (Marlin)
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
if quant_info.expert_map is not None:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""No-sort B200 decode kernels for Inkling routed MoE LoRA.
|
||||
|
||||
The contract is BF16 CUDA, M<=32, top-k 6, rank 32, up to four slots, TP-local
|
||||
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)``.
|
||||
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)`` or the inactive sentinel ``-1``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -47,7 +47,8 @@ def _direct_gate_expand_kernel(
|
||||
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_r = tl.arange(0, RANK)
|
||||
@@ -110,7 +111,8 @@ def _direct_down_shrink_kernel(
|
||||
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_r = tl.arange(0, RANK)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
@@ -121,7 +123,7 @@ def _direct_down_shrink_kernel(
|
||||
k_mask = k < INTERMEDIATE_SIZE
|
||||
x = tl.load(
|
||||
activation_ptr + route * stride_xq + k[None, :] * stride_xk,
|
||||
mask=(offs_m[:, None] == 0) & k_mask[None, :],
|
||||
mask=active & (offs_m[:, None] == 0) & k_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
a = tl.load(
|
||||
|
||||
@@ -232,8 +232,8 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
if (
|
||||
topk_ids.ndim != 2
|
||||
or topk_weights.shape != topk_ids.shape
|
||||
@@ -345,7 +345,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
!= lora_info.gate_up_lora_b_weights.shape[-1]
|
||||
)
|
||||
# Allocate and prewarm on main; side streams launch kernels only.
|
||||
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
|
||||
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
|
||||
if not direct_decode:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=gate_up_delta,
|
||||
@@ -572,9 +572,13 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
)
|
||||
|
||||
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -657,8 +661,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
zero_intermediate=(
|
||||
ep_active
|
||||
and lora_info.experts_shared_outer_loras
|
||||
lora_info.experts_shared_outer_loras
|
||||
and not factored_shared_outer
|
||||
),
|
||||
)
|
||||
@@ -679,11 +682,6 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: down (marlin).
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
if ep_active:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
@@ -764,7 +762,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
routing_cache=routing_cache,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
zero_intermediate=(ep_active and lora_info.experts_shared_outer_loras),
|
||||
zero_intermediate=lora_info.experts_shared_outer_loras,
|
||||
)
|
||||
|
||||
# Never alias hidden_states: the shared sink still reads it. The fused B200
|
||||
@@ -866,9 +864,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
down_intermediate if down_shrink_done is not None else None
|
||||
),
|
||||
zero_intermediate=(
|
||||
ep_active
|
||||
and lora_info.experts_shared_outer_loras
|
||||
and down_shrink_done is None
|
||||
lora_info.experts_shared_outer_loras and down_shrink_done is None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -835,7 +835,9 @@ class CPUGraphRunner:
|
||||
spec_algorithm=self.model_runner.spec_algorithm,
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=self.capture_hidden_mode,
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
num_token_non_padded=(
|
||||
self.num_token_non_padded if enable_num_token_non_padded() else None
|
||||
),
|
||||
global_forward_mode=self.capture_forward_mode,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
@@ -849,7 +851,8 @@ class CPUGraphRunner:
|
||||
)
|
||||
with skip_ctx:
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph(
|
||||
bs,
|
||||
@@ -991,11 +994,15 @@ class CPUGraphRunner:
|
||||
with replay_context():
|
||||
with skip_ctx:
|
||||
prepared_forward_batch = self.prepare_replay(forward_batch, skip=skip)
|
||||
output = graphs[prepared_forward_batch.batch_size](
|
||||
prepared_forward_batch.input_ids,
|
||||
prepared_forward_batch.positions,
|
||||
prepared_forward_batch,
|
||||
)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=prepared_forward_batch,
|
||||
):
|
||||
output = graphs[prepared_forward_batch.batch_size](
|
||||
prepared_forward_batch.input_ids,
|
||||
prepared_forward_batch.positions,
|
||||
prepared_forward_batch,
|
||||
)
|
||||
if forward_batch.batch_size in graphs:
|
||||
return output
|
||||
|
||||
|
||||
@@ -1143,14 +1143,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
batch.extend_input_logprob_token_ids.to(device, non_blocking=True)
|
||||
)
|
||||
|
||||
num_tokens = len(batch.input_ids) if batch.input_ids is not None else 0
|
||||
if enable_num_token_non_padded():
|
||||
ret.global_num_token_non_padded = torch.tensor(
|
||||
num_tokens,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(device),
|
||||
).to(device, non_blocking=True)
|
||||
ret.global_num_token_non_padded_cpu = num_tokens
|
||||
ret._init_num_token_non_padded(
|
||||
device=device, is_draft_worker=model_runner.is_draft_worker
|
||||
)
|
||||
|
||||
ret.init_mlp_sync_metadata(
|
||||
batch, device, is_draft_worker=model_runner.is_draft_worker
|
||||
@@ -1341,6 +1336,26 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
pin_memory=is_pin_memory_available(batch.device),
|
||||
).to(batch.device, non_blocking=True)
|
||||
|
||||
def _init_num_token_non_padded(self, *, device, is_draft_worker: bool) -> None:
|
||||
if self.input_ids is not None:
|
||||
num_tokens = len(self.input_ids)
|
||||
elif (
|
||||
is_draft_worker
|
||||
and self.spec_info is not None
|
||||
and self.forward_mode.is_decode()
|
||||
):
|
||||
assert self.spec_info.num_tokens_per_req > 0
|
||||
num_tokens = self.batch_size * self.spec_info.num_tokens_per_req
|
||||
else:
|
||||
num_tokens = 0
|
||||
if enable_num_token_non_padded():
|
||||
self.global_num_token_non_padded = torch.tensor(
|
||||
num_tokens,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(device),
|
||||
).to(device, non_blocking=True)
|
||||
self.global_num_token_non_padded_cpu = num_tokens
|
||||
|
||||
def set_local_num_token_non_padded(self, *, sharded: bool) -> None:
|
||||
"""Derive the LOCAL num_token_non_padded from the invariant GLOBAL scalar.
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ layer reads at depth via get_forward_context(). The only mandatory field
|
||||
today is attn_backend; pool refs are derived from attn_backend.*
|
||||
(every backend caches them at __init__), so a published ForwardContext
|
||||
is enough to resolve the active pools without a separate global.
|
||||
The active batch lends its MoE bound getter so late preparation and TBO child
|
||||
selection stay visible without retaining a parent tensor snapshot.
|
||||
|
||||
ModelRunner._forward_raw publishes a fresh ForwardContext for the
|
||||
duration of each forward; callers that need a per-call override (PDmux
|
||||
@@ -23,12 +25,15 @@ worker threads ever share a process, migrate to contextvars.ContextVar.
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -38,6 +43,8 @@ class ForwardContext:
|
||||
write time — use dataclasses.replace for per-call overrides."""
|
||||
|
||||
attn_backend: AttentionBackend
|
||||
# Resolve after batch preparation; captured calls retain the static buffer.
|
||||
moe_num_token_non_padded: Optional[Callable[[], Optional[torch.Tensor]]] = None
|
||||
|
||||
|
||||
_current: Optional[ForwardContext] = None
|
||||
@@ -76,7 +83,15 @@ def get_req_to_token_pool() -> ReqToTokenPool:
|
||||
|
||||
|
||||
@contextmanager
|
||||
def forward_context(ctx: ForwardContext):
|
||||
def forward_context(
|
||||
ctx: ForwardContext, *, forward_batch: Optional[ForwardBatch] = None
|
||||
):
|
||||
if forward_batch is not None:
|
||||
ctx = replace(
|
||||
ctx, moe_num_token_non_padded=forward_batch.moe_num_token_non_padded
|
||||
)
|
||||
elif ctx.moe_num_token_non_padded is None and _current is not None:
|
||||
ctx = replace(ctx, moe_num_token_non_padded=_current.moe_num_token_non_padded)
|
||||
prev = set_forward_context(ctx)
|
||||
try:
|
||||
yield
|
||||
|
||||
@@ -113,6 +113,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.graph_memory_usage import (
|
||||
@@ -1965,9 +1966,14 @@ class ModelRunner:
|
||||
if prepare is not None:
|
||||
prepare(forward_batch)
|
||||
if has_forward_context():
|
||||
ctx_mgr = contextlib.nullcontext()
|
||||
ctx_mgr = forward_context(
|
||||
get_forward_context(), forward_batch=forward_batch
|
||||
)
|
||||
else:
|
||||
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
|
||||
ctx_mgr = forward_context(
|
||||
ForwardContext(attn_backend=self.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
with (
|
||||
get_model().override(shared_experts=self.model.shared_experts_config),
|
||||
ctx_mgr,
|
||||
|
||||
@@ -762,7 +762,9 @@ class BaseRunner(ABC):
|
||||
|
||||
torch.get_device_module(mr.device).synchronize()
|
||||
get_parallel().tp_group.barrier()
|
||||
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=mr.attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
with run_ctx or empty_context():
|
||||
run_once()
|
||||
|
||||
|
||||
@@ -1253,7 +1253,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# All setup hooks below read get_attn_backend() (TboForwardBatchPreparer,
|
||||
# DeepEP adapter, …) so they must run inside the same ForwardContext
|
||||
# that wraps the warmup/capture forward.
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens)
|
||||
|
||||
if forward_batch.lora_ids is not None:
|
||||
@@ -1262,60 +1264,67 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
|
||||
|
||||
def run_once():
|
||||
# Backends may gather live pool mappings into their static
|
||||
# metadata here; these nodes run before any model KV reads.
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
self._record_in_graph_metadata_prep_done()
|
||||
|
||||
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
|
||||
None
|
||||
)
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
forward_batch.global_num_tokens_cpu,
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
kwargs = {}
|
||||
if (
|
||||
self.pp_size > 1
|
||||
and "pp_proxy_tensors" in inspect.signature(forward).parameters
|
||||
# Debug replay retains this callback beyond the capture scope.
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
kwargs["pp_proxy_tensors"] = PPProxyTensors(
|
||||
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
|
||||
)
|
||||
if (
|
||||
(
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
if self.dllm_input_preparation is None:
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
|
||||
else:
|
||||
state = self.buffer_registry.get_slot(
|
||||
"dllm_input_preparation_state"
|
||||
).buffer[:num_tokens]
|
||||
kwargs["input_embeds"] = self.dllm_input_preparation(
|
||||
self.model_runner.model, forward_batch.input_ids, state
|
||||
)
|
||||
# Backends may gather live pool mappings into their static
|
||||
# metadata here; these nodes run before any model KV reads.
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
self._record_in_graph_metadata_prep_done()
|
||||
|
||||
out = forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
for capture_hook in self.model_runner.capture_tail_hooks:
|
||||
capture_hook(self, out, forward_batch, num_tokens)
|
||||
return out
|
||||
forward_batch.dp_local_start_pos = (
|
||||
forward_batch.dp_local_num_tokens
|
||||
) = None
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
forward_batch.global_num_tokens_cpu,
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
kwargs = {}
|
||||
if (
|
||||
self.pp_size > 1
|
||||
and "pp_proxy_tensors" in inspect.signature(forward).parameters
|
||||
):
|
||||
kwargs["pp_proxy_tensors"] = PPProxyTensors(
|
||||
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
|
||||
)
|
||||
if (
|
||||
(
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
if self.dllm_input_preparation is None:
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[
|
||||
:num_tokens
|
||||
]
|
||||
else:
|
||||
state = self.buffer_registry.get_slot(
|
||||
"dllm_input_preparation_state"
|
||||
).buffer[:num_tokens]
|
||||
kwargs["input_embeds"] = self.dllm_input_preparation(
|
||||
self.model_runner.model, forward_batch.input_ids, state
|
||||
)
|
||||
|
||||
out = forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
for capture_hook in self.model_runner.capture_tail_hooks:
|
||||
capture_hook(self, out, forward_batch, num_tokens)
|
||||
return out
|
||||
|
||||
self.deepep_adapter.capture(is_extend_in_batch=False)
|
||||
canary_ctx = (
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
get_req_to_token_pool,
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
@@ -259,7 +260,11 @@ class EagerRunner(BaseRunner):
|
||||
|
||||
ctx = device_timer_ctx(model_runner.device_timer, "decode")
|
||||
|
||||
with ctx, pdmux_ctx:
|
||||
with (
|
||||
ctx,
|
||||
pdmux_ctx,
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
return model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
@@ -341,7 +346,10 @@ class EagerRunner(BaseRunner):
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
else "extend"
|
||||
)
|
||||
with device_timer_ctx(model_runner.device_timer, category):
|
||||
with (
|
||||
device_timer_ctx(model_runner.device_timer, category),
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
if cp_active and (
|
||||
not get_parallel().enable_cp_tp_group_sharing
|
||||
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
|
||||
@@ -448,7 +456,10 @@ class EagerRunner(BaseRunner):
|
||||
model_runner.attn_backend.forward_metadata = None
|
||||
|
||||
kwargs = model_runner._pp_kwargs(pp_proxy_tensors)
|
||||
with device_timer_ctx(model_runner.device_timer, "idle"):
|
||||
with (
|
||||
device_timer_ctx(model_runner.device_timer, "idle"),
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
return model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
|
||||
@@ -819,7 +819,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
full_graph=self._is_full_backend, raw_num_tokens=raw_num_tokens
|
||||
),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
set_tc_piecewise_forward_context(
|
||||
forward_batch,
|
||||
@@ -953,7 +954,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
set_forward_batch(fb),
|
||||
prefill_graph_scope(full_graph=self._is_full_backend),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=fb,
|
||||
),
|
||||
set_tc_piecewise_forward_context(
|
||||
fb,
|
||||
@@ -1234,7 +1236,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
and stash the returned per-bucket metadata object; otherwise fall
|
||||
back to the generic eager init that BCG/TC_PIECEWISE use today."""
|
||||
attn_backend = self.model_runner.attn_backend
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
if not self.use_captured_attn_metadata:
|
||||
attn_backend.init_forward_metadata(forward_batch)
|
||||
return
|
||||
|
||||
@@ -20,7 +20,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
set_graph_pool_id,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
forward_context,
|
||||
get_attn_backend,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
BreakableCUDAGraph,
|
||||
BreakableCUDAGraphCapture,
|
||||
@@ -120,7 +125,14 @@ class RowBucketGraphRunner:
|
||||
# The eager breaks read the step's real row count off this view.
|
||||
step_batch = copy.copy(forward_batch)
|
||||
step_batch.global_num_token_non_padded_cpu = num_rows
|
||||
with context(step_batch):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=step_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
context(step_batch),
|
||||
):
|
||||
if (rows, graph_key) not in self._graphs:
|
||||
# Largest first, so smaller buckets reuse the pool memory the
|
||||
# larger ones took; a bucket below this step's rows cannot be
|
||||
@@ -168,7 +180,14 @@ class RowBucketGraphRunner:
|
||||
forward_batch.num_token_non_padded, num_rows
|
||||
)
|
||||
capture_batch.num_token_non_padded = num_token_non_padded
|
||||
with batch_scope(capture_batch, inputs):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=capture_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
batch_scope(capture_batch, inputs),
|
||||
):
|
||||
graph = self._record(
|
||||
lambda: body(inputs, capture_batch),
|
||||
rows,
|
||||
|
||||
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -492,10 +491,6 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
hidden_states,
|
||||
router_logits,
|
||||
dynamic_expert_bias=dynamic_expert_bias,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -21,7 +21,6 @@ from sglang.srt.configs import KimiLinearConfig
|
||||
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
@@ -832,10 +831,6 @@ class BailingMoE(nn.Module):
|
||||
hidden_states,
|
||||
router_logits,
|
||||
dynamic_expert_bias=dynamic_expert_bias,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
|
||||
@@ -66,7 +66,6 @@ from sglang.srt.distributed import divide
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.amx_utils import PackWeightMethod
|
||||
@@ -120,10 +119,10 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
|
||||
DispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
BypassedTopKOutput,
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
fused_shared_expert_weight,
|
||||
is_expert_remapping_enabled,
|
||||
is_routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
@@ -1038,7 +1037,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
and not self._enable_a2a_moe
|
||||
and not self._fuse_shared_experts_inside_sbo
|
||||
and not getattr(self, "is_hash", False)
|
||||
and not get_exec().moe.enable_eplb
|
||||
and not is_expert_remapping_enabled()
|
||||
and get_exec().moe.ep_dispatch_algorithm not in ("fake", "lp")
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1067,11 +1067,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
|
||||
num_token_non_padded = (
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
|
||||
if use_vision_topk and _is_hip:
|
||||
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
|
||||
@@ -1109,7 +1104,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
gemm_output_zero_allocator,
|
||||
input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
@@ -1121,7 +1115,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
skip_shared_experts=skip_shared_experts,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
@@ -1137,7 +1130,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
@@ -1167,11 +1159,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
has_shared_output = (
|
||||
hidden_states.shape[0] > 0 and self.num_fused_shared_experts == 0
|
||||
)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb and not self.is_nextn
|
||||
else None
|
||||
)
|
||||
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
if _is_hip:
|
||||
@@ -1185,11 +1172,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
router_logits_partials = None
|
||||
if use_flashinfer_trtllm_bypass:
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
topk_config=self.topk.topk_config,
|
||||
)
|
||||
topk_output = self.topk.forward_cuda(hidden_states, router_logits)
|
||||
else:
|
||||
topk_kwargs = (
|
||||
{"input_ids": input_ids_global}
|
||||
@@ -1203,14 +1186,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
self,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
# Issued after the router so the main chain stays on the main stream at replay.
|
||||
@@ -1323,7 +1303,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
skip_shared_experts: bool = False,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
@@ -1332,11 +1311,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.shared_experts.gate_up_proj
|
||||
):
|
||||
return self.forward_cpu(hidden_states)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb and not self.is_nextn
|
||||
else None
|
||||
)
|
||||
defer_shared = not self.experts.moe_runner_config.inplace
|
||||
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): shared expert is computed on the LOCAL
|
||||
# hidden in the decoder layer (before the dp gather) and added after the
|
||||
@@ -1383,21 +1357,18 @@ class DeepseekV2MoE(nn.Module):
|
||||
self,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
pre_quant_input = None
|
||||
shared_output = None
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
if self._fuse_shared_experts_inside_sbo and not skip_shared_experts:
|
||||
@@ -1591,19 +1562,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
if sbo_overlap_dispatch_flag:
|
||||
@@ -1979,19 +1942,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
|
||||
@@ -28,7 +28,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
forward_context,
|
||||
get_attn_backend,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
compute_attention_and_moe_layers,
|
||||
)
|
||||
@@ -163,6 +168,15 @@ class DecoderReplayTail:
|
||||
max_rows=max_rows,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _can_capture_moe_bound(forward_batch: ForwardBatch) -> bool:
|
||||
# The row runner replaces only the local count with a graph-owned buffer.
|
||||
bound = forward_batch.moe_num_token_non_padded()
|
||||
return bound is None or (
|
||||
bound is forward_batch.num_token_non_padded
|
||||
and bound is not forward_batch.global_num_token_non_padded
|
||||
)
|
||||
|
||||
def plan(
|
||||
self,
|
||||
tail: Optional[LateLayerTail],
|
||||
@@ -199,6 +213,7 @@ class DecoderReplayTail:
|
||||
or get_global_expert_distribution_recorder().recording
|
||||
or not self.runner.warm
|
||||
or (tail is not None and tail.cp_metadata is not None)
|
||||
or not self._can_capture_moe_bound(forward_batch)
|
||||
):
|
||||
return False, None
|
||||
if not dp:
|
||||
@@ -414,7 +429,14 @@ class DecoderReplayTail:
|
||||
gathered = input_ids.new_empty((sum(dp_rows), 1))
|
||||
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
|
||||
input_ids_global = gathered.squeeze(-1)
|
||||
with self._break_context(tail_batch):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=tail_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
self._break_context(tail_batch),
|
||||
):
|
||||
return self._layer_stack(
|
||||
state,
|
||||
positions=positions,
|
||||
@@ -681,6 +703,7 @@ def _tail_layers_in_prefill_graph(
|
||||
if (
|
||||
not get_flags().dp.capturing_prefill_graph
|
||||
and runner.warm
|
||||
and tail_layers._can_capture_moe_bound(forward_batch)
|
||||
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
|
||||
):
|
||||
residual, pre, aux = tail_layers.run(**kwargs)
|
||||
|
||||
@@ -51,7 +51,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
@@ -467,14 +466,10 @@ class Dots3MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -26,7 +26,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -228,10 +227,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -653,10 +652,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -30,7 +30,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -377,10 +376,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -34,7 +34,6 @@ from triton.language.extra import libdevice
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -918,10 +917,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -97,6 +97,7 @@ from sglang.srt.model_loader.utils import (
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_stream,
|
||||
)
|
||||
@@ -251,6 +252,15 @@ class LongcatFlashMoE(nn.Module):
|
||||
self.top_k = config.moe_topk
|
||||
self.zero_expert_num = config.zero_expert_num
|
||||
self.zero_expert_type = config.zero_expert_type
|
||||
if (
|
||||
self.zero_expert_num > 0
|
||||
and get_exec().moe.ep_dispatch_algorithm is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"LongCat zero experts do not support expert placement remapping. "
|
||||
"Disable EPLB, redundant experts, custom expert placement, and "
|
||||
"--ep-dispatch-algorithm for this model."
|
||||
)
|
||||
|
||||
if config.rounter_params_dtype == "float32":
|
||||
self.rounter_params_dtype = torch.float32
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -559,11 +558,6 @@ class MiMoV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if self._uses_expert_remap
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -595,10 +589,6 @@ class MiMoV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -628,10 +618,6 @@ class MiMoV2MoE(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
@@ -614,10 +613,6 @@ class MiniMaxM2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(device=hidden_states.device)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.configs.model_config import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -535,10 +534,6 @@ class MiniMaxM3MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import ReLU2
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
HybridLinearAttnBackend,
|
||||
@@ -349,7 +348,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if (
|
||||
@@ -364,21 +362,16 @@ class NemotronHMoE(nn.Module):
|
||||
and is_in_breakable_cuda_graph()
|
||||
and is_sm120()
|
||||
):
|
||||
return self._forward_core_prefill_overlap(
|
||||
hidden_states, num_token_non_padded
|
||||
)
|
||||
return self._forward_core_prefill_overlap(hidden_states)
|
||||
if _is_cuda and (
|
||||
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
|
||||
):
|
||||
return self._forward_core_shared_routed_overlap(
|
||||
hidden_states, num_token_non_padded
|
||||
)
|
||||
return self._forward_core_normal(hidden_states, num_token_non_padded)
|
||||
return self._forward_core_shared_routed_overlap(hidden_states)
|
||||
return self._forward_core_normal(hidden_states)
|
||||
|
||||
def _forward_core_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# router_scores: [num_tokens, num_experts]
|
||||
# bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk.
|
||||
@@ -392,12 +385,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -407,7 +394,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core_shared_routed_overlap(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
alt_stream = _get_or_create_alt_stream(self.device_module)
|
||||
|
||||
@@ -427,12 +413,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -444,7 +424,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core_prefill_overlap(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# A multi-stream graph per hybrid segment creates one internal stream
|
||||
# per executable. Keep each graph linear and launch shared experts on
|
||||
@@ -460,12 +439,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -509,11 +482,6 @@ class NemotronHMoE(nn.Module):
|
||||
# MoE runner / topk), so final_hidden_states is already scaled.
|
||||
final_hidden_states, shared_output = self._forward_core(
|
||||
hidden_states,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -611,14 +610,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -667,16 +658,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.enable_shared_expert_fusion:
|
||||
topk_output = self._append_shared_to_topk_output(
|
||||
|
||||
@@ -28,7 +28,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
@@ -342,20 +341,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb
|
||||
else None
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] > 0:
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
)
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
@@ -370,10 +359,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -408,14 +393,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
assert TopKOutputChecker.format_is_standard(topk_output), (
|
||||
"MegaMoE pre-dispatch consumes raw topk ids/weights; "
|
||||
@@ -454,10 +431,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -13,7 +13,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -159,10 +158,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -6,7 +6,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -256,10 +255,6 @@ class Step3p5MoEMLP(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -38,7 +38,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
@@ -1000,10 +999,6 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
),
|
||||
)
|
||||
# Apply router scaling factor after renormalization
|
||||
if self.router_scaling_factor != 1.0:
|
||||
@@ -1014,7 +1009,7 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
topk_output.topk_weights = scaled_weights
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
@@ -1051,7 +1046,7 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
if hidden_states.shape[0] == 0:
|
||||
shared_output = None
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
else:
|
||||
|
||||
@@ -2,6 +2,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import transform_select_experts_inputs
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
_RENORMALIZE_SUM_EPSILON,
|
||||
StandardTopKOutput,
|
||||
@@ -12,19 +13,10 @@ from sglang.srt.layers.moe.topk import (
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
|
||||
def _scale_fused_shared_weights(weights, num_fused_shared_experts, scaling_factor):
|
||||
# Standard EP replicates the fused shared expert on every rank and all-reduces,
|
||||
# so the shared columns carry a 1/ep_size factor.
|
||||
if num_fused_shared_experts and scaling_factor is not None:
|
||||
weights[:, -num_fused_shared_experts:] *= scaling_factor
|
||||
return weights
|
||||
|
||||
|
||||
def vision_topk(
|
||||
moe,
|
||||
logits,
|
||||
@@ -32,25 +24,35 @@ def vision_topk(
|
||||
num_token_non_padded=None,
|
||||
expert_location_dispatch_info=None,
|
||||
):
|
||||
bound, dispatch_info = moe.topk.routing_inputs()
|
||||
if num_token_non_padded is None:
|
||||
num_token_non_padded = bound
|
||||
if expert_location_dispatch_info is None:
|
||||
expert_location_dispatch_info = dispatch_info
|
||||
config = moe.topk.topk_config
|
||||
logits, correction_bias = transform_select_experts_inputs(
|
||||
logits, moe.gate.e_score_correction_bias, expert_location_dispatch_info
|
||||
)
|
||||
correction_bias_vl = moe.gate.e_score_correction_bias_vl
|
||||
if (
|
||||
expert_location_dispatch_info is not None
|
||||
and expert_location_dispatch_info.ep_dispatch_algorithm == "fake"
|
||||
):
|
||||
correction_bias_vl = torch.zeros_like(correction_bias_vl)
|
||||
num_shared_topk_slots = config.num_shared_topk_slots
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if num_shared_topk_slots and not use_mega_moe:
|
||||
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
|
||||
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
|
||||
"VL routing does not support per-rank fused shared slots"
|
||||
)
|
||||
if is_cuda():
|
||||
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
|
||||
from sglang.srt.layers.moe.utils import get_moe_runner_backend
|
||||
|
||||
# Same admission as _fused_gate_emits_packed_ids: the shared-expert slots
|
||||
# rescaled below would rewrite weights after the router.
|
||||
packed_topk = None
|
||||
if (
|
||||
num_shared_topk_slots == 0
|
||||
and expert_location_dispatch_info is None
|
||||
and not use_mega_moe
|
||||
and get_moe_runner_backend().is_flashinfer_mxfp4()
|
||||
and config.runner_backend is not None
|
||||
and config.runner_backend.is_flashinfer_mxfp4()
|
||||
):
|
||||
packed_topk = torch.empty(
|
||||
(logits.shape[0], config.top_k), dtype=torch.int32, device=logits.device
|
||||
@@ -58,11 +60,11 @@ def vision_topk(
|
||||
|
||||
weights, indices = moe_fused_gate(
|
||||
logits,
|
||||
moe.gate.e_score_correction_bias,
|
||||
correction_bias,
|
||||
topk=config.top_k,
|
||||
scoring_func="sqrtsoftplus",
|
||||
num_fused_shared_experts=num_shared_topk_slots,
|
||||
bias_alt=moe.gate.e_score_correction_bias_vl,
|
||||
bias_alt=correction_bias_vl,
|
||||
input_ids=input_ids,
|
||||
bias_alt_token_id=moe.config.image_token_id,
|
||||
renormalize=config.renormalize and config.top_k > 1,
|
||||
@@ -87,12 +89,12 @@ def vision_topk(
|
||||
return StandardTopKOutput(weights, indices, logits)
|
||||
scores = F.softplus(logits.float()).sqrt()
|
||||
if input_ids is None:
|
||||
bias = moe.gate.e_score_correction_bias
|
||||
bias = correction_bias
|
||||
else:
|
||||
bias = torch.where(
|
||||
(input_ids == moe.config.image_token_id)[:, None],
|
||||
moe.gate.e_score_correction_bias_vl,
|
||||
moe.gate.e_score_correction_bias,
|
||||
correction_bias_vl,
|
||||
correction_bias,
|
||||
)
|
||||
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
|
||||
topk_routed = config.top_k - num_shared_topk_slots
|
||||
@@ -136,28 +138,19 @@ def _finish_vision_topk(
|
||||
):
|
||||
config = moe.topk.topk_config
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if use_mega_moe:
|
||||
# Use the same EPLB mapping, recording and home-rank shared slots as
|
||||
# ordinary top-k before handing physical expert IDs to MegaMoE.
|
||||
indices, weights, recorder_ids = _post_process_topk_ids(
|
||||
indices,
|
||||
weights,
|
||||
config,
|
||||
logits,
|
||||
moe.layer_id,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
else:
|
||||
weights = _scale_fused_shared_weights(
|
||||
weights,
|
||||
config.num_shared_topk_slots,
|
||||
config.fused_shared_experts_scaling_factor,
|
||||
indices, weights, recorder_ids = _post_process_topk_ids(
|
||||
indices,
|
||||
weights,
|
||||
config,
|
||||
logits,
|
||||
moe.layer_id,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
# Shared-slot remapping can overwrite padded weights; mask last so idle
|
||||
# and partially padded DP ranks never dispatch a phantom shared expert.
|
||||
|
||||
@@ -511,7 +511,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
|
||||
return ret
|
||||
|
||||
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.draft_attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
|
||||
@@ -482,7 +482,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
return ret
|
||||
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_extend_attn_backend)
|
||||
ForwardContext(attn_backend=self.draft_extend_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.draft_extend_attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
|
||||
@@ -800,7 +800,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
forward_context(
|
||||
ForwardContext(
|
||||
attn_backend=self.draft_attn_backend.attn_backends[i]
|
||||
)
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
canary_index_ctx,
|
||||
):
|
||||
@@ -903,7 +904,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
forward_context(
|
||||
ForwardContext(
|
||||
attn_backend=self.draft_attn_backend.attn_backends[i]
|
||||
)
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
canary_index_ctx,
|
||||
):
|
||||
|
||||
@@ -357,7 +357,10 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
saved_backend_pool = self.draft_attn_backend.token_to_kv_pool
|
||||
self.draft_attn_backend.token_to_kv_pool = target_pool
|
||||
try:
|
||||
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
@@ -565,7 +565,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
|
||||
with (
|
||||
self._target_kv_pool_view(forward_batch),
|
||||
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
):
|
||||
seed_output = self.draft_model_runner.forward(forward_batch).logits_output
|
||||
|
||||
@@ -608,7 +611,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
|
||||
with (
|
||||
self._target_kv_pool_view(forward_batch),
|
||||
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
):
|
||||
logits_output = self.draft_model_runner.forward(
|
||||
forward_batch
|
||||
|
||||
@@ -451,7 +451,9 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
return self._run_step_body(forward_batch, num_tokens, bs)
|
||||
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
|
||||
self.deepep_adapter.capture(is_extend_in_batch=True)
|
||||
shape_key = self._make_graph_key(bs)
|
||||
@@ -1026,7 +1028,9 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
forward_batch = r.get_forward_batch(bs)
|
||||
forward_batch = r._postprocess_forward_batch(forward_batch, bs)
|
||||
attn_backend = self.draft_extend_attn_backend_list[r.step]
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
@@ -1037,7 +1041,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
outs = []
|
||||
for i, (r, forward_batch, num_tokens, attn_backend) in enumerate(items):
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
ret = r._run_step_body(forward_batch, num_tokens, bs)
|
||||
outs.append(ret)
|
||||
if i < n - 1:
|
||||
|
||||
@@ -814,7 +814,10 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
attn_context = (
|
||||
forward_context(ForwardContext(attn_backend=self._uno_draft_attn_backend))
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self._uno_draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if self.tree_mode
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
"""CUTLASS FP8 maps and reductions skip inactive expert routes."""
|
||||
|
||||
import unittest
|
||||
from collections import defaultdict
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.shuffle_weighted_sum import (
|
||||
_shuffle_weighted_sum_kernel,
|
||||
shuffle_weighted_sum,
|
||||
)
|
||||
from sglang.srt.layers.moe import cutlass_moe
|
||||
from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=12, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestCutlassFp8InactiveRoutes(CustomTestCase):
|
||||
def test_masked_reduction_ignores_poisoned_slots(self):
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
with self.subTest(dtype=dtype):
|
||||
inputs = torch.full((6, 257), float("nan"), device="cuda", dtype=dtype)
|
||||
inputs[0].fill_(2)
|
||||
inputs[1].fill_(4)
|
||||
mapping = torch.tensor(
|
||||
[0, 1, -1, -1, 0, -1], device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[[0.25, 0.75], [float("nan"), float("nan")], [0.5, float("nan")]],
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
)
|
||||
output = torch.empty((3, 257), device="cuda", dtype=dtype)
|
||||
shuffle_weighted_sum(inputs, output, mapping, weights)
|
||||
expected = torch.tensor([3.5, 0, 1], device="cuda", dtype=dtype)[
|
||||
:, None
|
||||
].expand_as(output)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
def test_reduction_reuses_kernels_across_token_counts(self):
|
||||
"""New lengths reuse the reduction and preserve mixed/empty route results."""
|
||||
kernel = _shuffle_weighted_sum_kernel
|
||||
lengths = (1, 3, 7, 8, 15, 16, 17, 31, 32, 33, 64, 65)
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
with (
|
||||
self.subTest(dtype=dtype),
|
||||
patch.object(
|
||||
kernel, "device_caches", defaultdict(kernel.create_binder)
|
||||
),
|
||||
):
|
||||
cache = kernel.device_caches[torch.cuda.current_device()][0]
|
||||
specialization_count = None
|
||||
for strided_weights in (False, True):
|
||||
for tokens in lengths:
|
||||
with self.subTest(
|
||||
tokens=tokens, strided_weights=strided_weights
|
||||
):
|
||||
copies = (tokens + 2) // 3
|
||||
inputs = torch.full(
|
||||
(tokens * 2, 257),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
)
|
||||
inputs[0].fill_(2)
|
||||
inputs[1].fill_(4)
|
||||
mapping = (
|
||||
torch.tensor(
|
||||
[[0, 1], [1, -1], [-1, -1]],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
.repeat(copies, 1)[:tokens]
|
||||
.flatten()
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[
|
||||
[0.25, 0.75],
|
||||
[0.5, float("nan")],
|
||||
[float("nan"), float("nan")],
|
||||
],
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
).repeat(copies, 1)[:tokens]
|
||||
if strided_weights:
|
||||
weights = weights.t().contiguous().t()
|
||||
output = torch.empty(
|
||||
(tokens, 257), device="cuda", dtype=dtype
|
||||
)
|
||||
shuffle_weighted_sum(inputs, output, mapping, weights)
|
||||
expected = (
|
||||
torch.tensor([3.5, 2, 0], device="cuda", dtype=dtype)
|
||||
.repeat(copies)[:tokens, None]
|
||||
.expand_as(output)
|
||||
)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
if specialization_count is None:
|
||||
specialization_count = len(cache)
|
||||
self.assertEqual(len(cache), specialization_count)
|
||||
|
||||
mapping.fill_(-1)
|
||||
inputs.fill_(float("nan"))
|
||||
weights.fill_(float("nan"))
|
||||
output.fill_(float("nan"))
|
||||
shuffle_weighted_sum(inputs, output, mapping, weights)
|
||||
self.assertTrue(bool(torch.isfinite(output).all()))
|
||||
self.assertEqual(int(torch.count_nonzero(output)), 0)
|
||||
self.assertEqual(len(cache), specialization_count)
|
||||
|
||||
def test_native_fp8_padding_survives_graph_replay(self):
|
||||
if torch.cuda.get_device_capability()[0] not in (9, 10, 12):
|
||||
self.skipTest("CUTLASS block FP8 requires SM90, SM100 or SM120")
|
||||
for mxfp8 in (False, True):
|
||||
if mxfp8 and torch.cuda.get_device_capability()[0] != 10:
|
||||
continue
|
||||
with self.subTest(mxfp8=mxfp8):
|
||||
self._check_native(mxfp8)
|
||||
|
||||
def _check_native(self, mxfp8):
|
||||
experts, tokens, hidden, intermediate = 2, 16, 256, 256
|
||||
generator = torch.Generator(device="cuda").manual_seed(945)
|
||||
inputs = (
|
||||
torch.randn(
|
||||
(tokens, hidden),
|
||||
generator=generator,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ 8
|
||||
)
|
||||
ids = torch.tensor(
|
||||
[[0, 1], [1, -1], [-1, -1], [1, 0]] * 4, device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[
|
||||
[0.25, 0.75],
|
||||
[0.6, float("nan")],
|
||||
[float("nan"), float("nan")],
|
||||
[0.8, 0.2],
|
||||
]
|
||||
* 4,
|
||||
device="cuda",
|
||||
)
|
||||
inputs[(ids < 0).all(dim=-1)] = float("nan")
|
||||
w1 = (
|
||||
(
|
||||
torch.randn(
|
||||
(experts, 2 * intermediate, hidden),
|
||||
generator=generator,
|
||||
device="cuda",
|
||||
)
|
||||
/ 16
|
||||
)
|
||||
.to(torch.float8_e4m3fn)
|
||||
.transpose(1, 2)
|
||||
)
|
||||
w2 = (
|
||||
(
|
||||
torch.randn(
|
||||
(experts, hidden, intermediate), generator=generator, device="cuda"
|
||||
)
|
||||
/ 16
|
||||
)
|
||||
.to(torch.float8_e4m3fn)
|
||||
.transpose(1, 2)
|
||||
)
|
||||
if mxfp8:
|
||||
scale1 = torch.full(
|
||||
(experts, hidden // 32, 2 * intermediate),
|
||||
127,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
scale2 = torch.full(
|
||||
(experts, intermediate // 32, hidden),
|
||||
127,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
else:
|
||||
scale1 = torch.ones(
|
||||
(experts, 2 * intermediate // 128, hidden // 128), device="cuda"
|
||||
).transpose(1, 2)
|
||||
scale2 = torch.ones(
|
||||
(experts, hidden // 128, intermediate // 128), device="cuda"
|
||||
).transpose(1, 2)
|
||||
params = CutlassMoEParams(
|
||||
CutlassMoEType.BlockscaledFP8,
|
||||
torch.device("cuda"),
|
||||
experts,
|
||||
intermediate,
|
||||
hidden,
|
||||
)
|
||||
workspace = torch.empty(32 * 1024 * 1024, device="cuda", dtype=torch.uint8)
|
||||
|
||||
def run(x, route_ids, route_weights):
|
||||
return cutlass_moe.cutlass_fused_experts_fp8(
|
||||
x,
|
||||
w1,
|
||||
w2,
|
||||
scale1,
|
||||
scale2,
|
||||
route_weights,
|
||||
route_ids,
|
||||
params.ab_strides_13,
|
||||
params.c_strides_13,
|
||||
params.ab_strides_2,
|
||||
params.c_strides_2,
|
||||
workspace,
|
||||
params.a_ptrs,
|
||||
params.b_ptrs,
|
||||
params.out_ptrs,
|
||||
params.a_scales_ptrs,
|
||||
params.b_scales_ptrs,
|
||||
params.expert_offsets,
|
||||
params.problem_sizes1,
|
||||
params.problem_sizes2,
|
||||
use_mxfp8=mxfp8,
|
||||
enable_es=(mxfp8, mxfp8),
|
||||
)
|
||||
|
||||
reference = run(inputs.nan_to_num(), ids.clamp_min(0), weights.nan_to_num())
|
||||
prepare = cutlass_moe.prepare_moe_input
|
||||
|
||||
def checked_prepare(*args):
|
||||
prepare(*args)
|
||||
route_ids, offsets, _, _, a_map, c_map = args[:6]
|
||||
self.assertTrue(bool(((a_map >= 0) & (a_map < tokens)).all()))
|
||||
self.assertTrue(bool((c_map[route_ids.flatten() < 0] == -1).all()))
|
||||
self.assertEqual(int(offsets[-1]), int((route_ids >= 0).sum()))
|
||||
|
||||
original_empty = torch.empty
|
||||
|
||||
def poisoned_empty(*args, **kwargs):
|
||||
tensor = original_empty(*args, **kwargs)
|
||||
if tensor.is_floating_point():
|
||||
tensor.fill_(float("nan"))
|
||||
elif tensor.dtype == torch.int32:
|
||||
tensor.fill_(-999)
|
||||
return tensor
|
||||
|
||||
# The pre-fix map allocation fails checked_prepare before an unsafe gather.
|
||||
with (
|
||||
patch.object(torch, "empty", poisoned_empty),
|
||||
patch.object(cutlass_moe, "prepare_moe_input", checked_prepare),
|
||||
):
|
||||
actual = run(inputs, ids, weights)
|
||||
torch.testing.assert_close(actual, reference, rtol=0.025, atol=0.01)
|
||||
self.assertTrue(bool(torch.isfinite(actual).all()))
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
result = run(inputs, ids, weights)
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
result = run(inputs, ids, weights)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(result, reference, rtol=0.025, atol=0.01)
|
||||
ids.fill_(-1)
|
||||
inputs.fill_(float("nan"))
|
||||
weights.fill_(float("nan"))
|
||||
graph.replay()
|
||||
self.assertTrue(bool(torch.isfinite(result).all()))
|
||||
self.assertEqual(int(torch.count_nonzero(result)), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,185 @@
|
||||
"""W4A8 routing must pack inactive routes after live experts, including at EP1."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.layers.moe import cutlass_w4a8_moe as w4a8
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=8, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
class TestCutlassW4A8InactiveRoutes(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not torch.cuda.is_available():
|
||||
raise unittest.SkipTest("CUDA is not available")
|
||||
|
||||
def test_ep1_and_ep2_reorder_match_dense_experts(self):
|
||||
"""Real CUDA permutation/count/finalize kernels must ignore padded routes.
|
||||
|
||||
Only the external CUTLASS GEMMs are replaced with dense expert GEMMs;
|
||||
this checks routing, not native INT4 GEMM numerics.
|
||||
"""
|
||||
experts, hidden, intermediate = 2, 128, 128
|
||||
ids = torch.tensor(
|
||||
[[1, 0], [0, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
|
||||
)
|
||||
inputs = (
|
||||
torch.arange(4 * hidden, device="cuda")
|
||||
.reshape(4, hidden)
|
||||
.remainder(7)
|
||||
.float()
|
||||
- 3
|
||||
) / 16
|
||||
inputs = inputs.to(torch.bfloat16)
|
||||
eye = torch.eye(hidden, device="cuda")
|
||||
dense_w13 = [
|
||||
torch.cat((eye * ((e + 1) / 4), eye * ((e + 2) / 4)))
|
||||
for e in range(experts)
|
||||
]
|
||||
dense_w2 = [eye * ((e + 1) / 2) for e in range(experts)]
|
||||
|
||||
def dense_reference(dynamic):
|
||||
scale1 = (
|
||||
inputs[(ids >= 0).any(dim=-1)].float().abs().amax() / 448.0
|
||||
if dynamic
|
||||
else 1.0
|
||||
)
|
||||
activated = {}
|
||||
for token in range(ids.shape[0]):
|
||||
for slot in range(ids.shape[1]):
|
||||
expert = int(ids[token, slot])
|
||||
if expert < 0:
|
||||
continue
|
||||
quantized = (inputs[token].float() / scale1).to(torch.float8_e4m3fn)
|
||||
gate_up = F.linear(
|
||||
quantized.float() * scale1, dense_w13[expert]
|
||||
).to(torch.bfloat16)
|
||||
gate, up = gate_up.float().chunk(2)
|
||||
activated[token, slot] = F.silu(gate) * up
|
||||
scale2 = (
|
||||
torch.stack(list(activated.values())).abs().amax().clamp_min(1e-10)
|
||||
/ 448.0
|
||||
if dynamic
|
||||
else 1.0
|
||||
)
|
||||
result = torch.zeros_like(inputs, dtype=torch.float32)
|
||||
for (token, slot), value in activated.items():
|
||||
expert = int(ids[token, slot])
|
||||
quantized = (value / scale2).to(torch.float8_e4m3fn)
|
||||
down = F.linear(quantized.float() * scale2, dense_w2[expert]).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
result[token] += down.float() * weights[token, slot]
|
||||
return result
|
||||
|
||||
w13_q = torch.zeros(
|
||||
(experts, 2 * intermediate, hidden // 2), device="cuda", dtype=torch.int8
|
||||
)
|
||||
w2_q = torch.zeros(
|
||||
(experts, hidden, intermediate // 2), device="cuda", dtype=torch.int8
|
||||
)
|
||||
scales = torch.ones((experts, 1), device="cuda")
|
||||
strides = [
|
||||
torch.zeros((experts, 3), device="cuda", dtype=torch.int64)
|
||||
for _ in range(8)
|
||||
]
|
||||
offsets = torch.empty(experts + 1, device="cuda", dtype=torch.int32)
|
||||
sizes1 = torch.empty((experts, 3), device="cuda", dtype=torch.int32)
|
||||
sizes2 = torch.empty_like(sizes1)
|
||||
|
||||
def dense_gemm(
|
||||
out, x, packed, input_scale, _weight_scale, starts, sizes, *args
|
||||
):
|
||||
out.fill_(float("nan"))
|
||||
matrices = dense_w13 if packed is w13_q else dense_w2
|
||||
for expert in range(experts):
|
||||
start, count = int(starts[expert]), int(sizes[expert, 1])
|
||||
if count:
|
||||
values = x[start : start + count].float() * input_scale
|
||||
out[start : start + count].copy_(F.linear(values, matrices[expert]))
|
||||
|
||||
def run(value, *, dynamic, route_ids=ids, route_weights=weights):
|
||||
return w4a8.cutlass_w4a8_moe(
|
||||
value,
|
||||
w13_q,
|
||||
w2_q,
|
||||
scales,
|
||||
scales,
|
||||
route_weights,
|
||||
route_ids,
|
||||
*strides,
|
||||
offsets,
|
||||
sizes1,
|
||||
sizes2,
|
||||
a1_scale=None if dynamic else torch.ones(1, device="cuda"),
|
||||
a2_scale=None if dynamic else torch.ones(1, device="cuda"),
|
||||
)
|
||||
|
||||
for ep_size in (1, 2):
|
||||
with (
|
||||
get_parallel().override(
|
||||
tp_size=ep_size,
|
||||
tp_rank=0,
|
||||
attn_tp_size=ep_size,
|
||||
attn_tp_rank=0,
|
||||
moe_ep_size=ep_size,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
),
|
||||
patch.object(w4a8, "cutlass_w4a8_moe_mm", side_effect=dense_gemm),
|
||||
):
|
||||
for dynamic in (False, True):
|
||||
reference = dense_reference(dynamic)
|
||||
for poison in (0.0, 1e6, float("inf"), float("nan")):
|
||||
with self.subTest(
|
||||
ep_size=ep_size, dynamic=dynamic, poison=poison
|
||||
):
|
||||
value = inputs.clone()
|
||||
value[2].fill_(poison)
|
||||
route_weights = weights.masked_fill(ids < 0, float("nan"))
|
||||
actual = run(
|
||||
value, dynamic=dynamic, route_weights=route_weights
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
offsets,
|
||||
torch.tensor(
|
||||
[0, 3, 5], device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
self.assertTrue(torch.isfinite(actual).all().item())
|
||||
torch.testing.assert_close(
|
||||
actual, reference.to(actual.dtype), rtol=1e-2, atol=1e-3
|
||||
)
|
||||
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
|
||||
inactive = torch.full_like(ids, -1)
|
||||
actual = run(
|
||||
torch.full_like(inputs, float("nan")),
|
||||
dynamic=True,
|
||||
route_ids=inactive,
|
||||
route_weights=torch.full_like(weights, float("nan")),
|
||||
)
|
||||
self.assertTrue(torch.isfinite(actual).all().item())
|
||||
self.assertEqual(torch.count_nonzero(actual).item(), 0)
|
||||
torch.testing.assert_close(
|
||||
ids,
|
||||
torch.tensor(
|
||||
[[1, 0], [0, -1], [-1, -1], [1, 0]],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -10,7 +10,6 @@ from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute_with_scale
|
||||
from sglang.srt.layers.moe.moe_runner import humming as humming_runner
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.quantization import humming_utils
|
||||
from sglang.srt.utils import get_device_sm
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -37,6 +36,8 @@ class TestHummingFp8Dispatch(CustomTestCase):
|
||||
raise unittest.SkipTest("Humming FP8 dispatch requires SM90")
|
||||
|
||||
def test_fp8_dispatch_preserves_group_scales(self):
|
||||
from sglang.srt.layers.quantization import humming_utils
|
||||
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
hidden_states = torch.randn(
|
||||
5,
|
||||
@@ -258,5 +259,106 @@ class TestHummingFp8Dispatch(CustomTestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestHummingInactiveRoutes(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not torch.cuda.is_available():
|
||||
raise unittest.SkipTest("CUDA is not available")
|
||||
|
||||
def test_indexed_finalize_skips_poisoned_routes_without_ep(self):
|
||||
from sglang.kernels.ops.moe.moe_fused_mul_sum import moe_fused_mul_sum
|
||||
|
||||
ids = torch.tensor(
|
||||
[[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
|
||||
)
|
||||
# A legal leading storage record makes a negative map lookup observable
|
||||
# without making the test depend on an out-of-allocation device read.
|
||||
map_storage = torch.tensor([0, 0, -1], device="cuda", dtype=torch.int32)
|
||||
for dtype in (torch.bfloat16, torch.float16):
|
||||
for expert_map in (None, map_storage[1:]):
|
||||
for poison_weights in (False, True):
|
||||
with self.subTest(
|
||||
dtype=dtype,
|
||||
mapped=expert_map is not None,
|
||||
poison_weights=poison_weights,
|
||||
):
|
||||
valid = ids >= 0
|
||||
if expert_map is not None:
|
||||
valid &= expert_map[ids.clamp_min(0)] >= 0
|
||||
values = (
|
||||
torch.arange(4 * 2 * 128, device="cuda")
|
||||
.reshape(4, 2, 128)
|
||||
.remainder(11)
|
||||
.to(dtype)
|
||||
)
|
||||
values[~valid] = float("nan")
|
||||
route_weights = weights.clone()
|
||||
if poison_weights:
|
||||
route_weights[~valid] = float("nan")
|
||||
expected = (
|
||||
(
|
||||
torch.where(valid.unsqueeze(-1), values.float(), 0)
|
||||
* torch.where(valid, route_weights, 0).unsqueeze(-1)
|
||||
)
|
||||
.sum(dim=1)
|
||||
.to(dtype)
|
||||
)
|
||||
actual = moe_fused_mul_sum(
|
||||
values,
|
||||
route_weights,
|
||||
topk_ids=ids,
|
||||
expert_map=expert_map,
|
||||
is_ep=False,
|
||||
)
|
||||
self.assertTrue(torch.isfinite(actual).all().item())
|
||||
torch.testing.assert_close(actual, expected)
|
||||
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
|
||||
|
||||
def test_grouped_permutation_skips_inactive_routes_without_ep(self):
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute, moe_unpermute
|
||||
|
||||
ids = torch.tensor(
|
||||
[[0, 1], [1, -1], [-1, -1], [1, 0]], device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[[0.25, 0.75], [0.6, 0.4], [0.3, 0.7], [0.8, 0.2]], device="cuda"
|
||||
)
|
||||
inputs = (
|
||||
torch.arange(4 * 128, device="cuda")
|
||||
.reshape(4, 128)
|
||||
.remainder(7)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
output = torch.full(
|
||||
(ids.numel(), 128), float("nan"), device="cuda", dtype=inputs.dtype
|
||||
)
|
||||
permuted, src2dst, offsets = moe_permute(
|
||||
inputs, ids, num_experts=2, is_ep=False, outputs=output
|
||||
)
|
||||
self.assertTrue((src2dst.view_as(ids)[ids < 0] < 0).all().item())
|
||||
torch.testing.assert_close(
|
||||
offsets, torch.tensor([0, 2, 5], device="cuda", dtype=torch.int32)
|
||||
)
|
||||
down = torch.full_like(permuted, float("nan"))
|
||||
reference = torch.zeros_like(inputs)
|
||||
for expert in range(2):
|
||||
start, end = int(offsets[expert]), int(offsets[expert + 1])
|
||||
down[start:end] = permuted[start:end] * (expert + 1)
|
||||
for token in range(ids.shape[0]):
|
||||
for slot in range(ids.shape[1]):
|
||||
expert = int(ids[token, slot])
|
||||
if expert >= 0:
|
||||
reference[token] += (
|
||||
inputs[token] * (expert + 1) * weights[token, slot]
|
||||
)
|
||||
actual = moe_unpermute(down, src2dst, ids, weights)
|
||||
self.assertTrue(torch.isfinite(actual).all().item())
|
||||
torch.testing.assert_close(actual, reference, rtol=1e-2, atol=1e-2)
|
||||
self.assertEqual(torch.count_nonzero(actual[2]).item(), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,17 +1,24 @@
|
||||
import itertools
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from sgl_kernel.scalar_type import scalar_types
|
||||
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
from sglang.srt.layers import zero_copy_context
|
||||
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.srt.layers.moe.fused_moe_triton.fused_marlin_moe import fused_marlin_moe
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config
|
||||
from sglang.srt.layers.quantization.marlin_utils_fp4 import (
|
||||
prepare_moe_nvfp4_layer_for_marlin,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
|
||||
from sglang.srt.utils.common import is_sm80_supported, is_sm90_supported
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_marlin_utils import (
|
||||
@@ -20,7 +27,7 @@ from sglang.test.test_marlin_utils import (
|
||||
marlin_quantize,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=12, stage="base-b", runner_config="1-gpu-large")
|
||||
register_cuda_ci(est_time=720, stage="base-b", runner_config="1-gpu-large")
|
||||
|
||||
|
||||
def _has_aot_moe_wna16_marlin_gemm() -> bool:
|
||||
@@ -896,5 +903,448 @@ def test_mxfp4_tp8_load_before_padding(tp_rank):
|
||||
)
|
||||
|
||||
|
||||
def _marlin_inactive_weights(experts, output_size, input_size, quant_type):
|
||||
tensors = _setup_moe_weights(
|
||||
min(experts, 8), output_size, input_size, quant_type, 128, False, torch.bfloat16
|
||||
)
|
||||
result = []
|
||||
for tensor in tensors:
|
||||
if tensor is None:
|
||||
result.append(None)
|
||||
continue
|
||||
tensor = tensor.repeat((experts // tensor.shape[0],) + (1,) * (tensor.ndim - 1))
|
||||
# A leading poisoned record makes negative expert indexing observable
|
||||
# without reading outside an allocation during the failing regression.
|
||||
storage = tensor.new_empty((experts + 1, *tensor.shape[1:]))
|
||||
storage[0].fill_(float("nan") if tensor.is_floating_point() else 0)
|
||||
storage[1:].copy_(tensor)
|
||||
result.append(storage[1:])
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rows,experts,intermediate",
|
||||
[(1, 8, 128), (4, 128, 128), (16, 8, 256), (64, 128, 128)],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"quant_type,num_bits",
|
||||
[(scalar_types.uint4b8, 4), (scalar_types.uint8b128, 8), (scalar_types.uint4, 4)],
|
||||
)
|
||||
def test_fused_marlin_moe_inactive_routes(
|
||||
rows, experts, intermediate, quant_type, num_bits
|
||||
):
|
||||
"""EP1 padded graph rows contribute zero through Marlin and its shared caches."""
|
||||
_check_marlin_inactive_routes(rows, experts, intermediate, quant_type, num_bits)
|
||||
|
||||
|
||||
def test_stock_marlin_lora_inactive_routes():
|
||||
if torch.cuda.get_device_capability()[0] < 9:
|
||||
pytest.skip("Stock Marlin LoRA requires SM90 or newer")
|
||||
_check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "stock_lora")
|
||||
|
||||
|
||||
def test_factored_marlin_lora_inactive_routes():
|
||||
if torch.cuda.get_device_capability()[0] < 9:
|
||||
pytest.skip("Experimental Marlin LoRA requires SM90 or newer")
|
||||
_check_marlin_inactive_routes(4, 8, 128, scalar_types.uint4b8, 4, "factored_lora")
|
||||
|
||||
|
||||
def _check_marlin_inactive_routes(
|
||||
rows, experts, intermediate, quant_type, num_bits, runner_kind="fused"
|
||||
):
|
||||
torch.manual_seed(43)
|
||||
hidden, topk = 256, 2
|
||||
with get_context().override_server_args(
|
||||
model_path="dummy",
|
||||
device="cuda",
|
||||
moe_runner_backend="auto",
|
||||
ep_dispatch_algorithm=None,
|
||||
enable_eplb=False,
|
||||
enable_waterfill=False,
|
||||
):
|
||||
flags = get_flags().moe
|
||||
flag_values = {name: getattr(flags, name) for name in flags.__struct_fields__}
|
||||
flag_values["in_speculative_scope"] = False
|
||||
with (
|
||||
flags.override(**flag_values),
|
||||
get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_dp_size=1,
|
||||
attn_cp_size=1,
|
||||
tp_group=None,
|
||||
),
|
||||
):
|
||||
initialize_moe_config()
|
||||
x = torch.randn((rows, hidden), device="cuda", dtype=torch.bfloat16)
|
||||
logits = torch.randn((rows, experts), device="cuda", dtype=torch.float32)
|
||||
w1_ref, w1, s1, z1, g1, p1 = _marlin_inactive_weights(
|
||||
experts, 2 * intermediate, hidden, quant_type
|
||||
)
|
||||
w2_ref, w2, s2, z2, g2, p2 = _marlin_inactive_weights(
|
||||
experts, hidden, intermediate, quant_type
|
||||
)
|
||||
router = TopK(
|
||||
top_k=topk,
|
||||
layer_id=0,
|
||||
renormalize=True,
|
||||
output_format=TopKOutputFormat.STANDARD,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
router.bind_runner_backend(MoeRunnerBackend.MARLIN)
|
||||
live = torch.tensor(rows, device="cuda", dtype=torch.int32)
|
||||
batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
batch_size=1,
|
||||
input_ids=torch.arange(rows, device="cuda"),
|
||||
req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64),
|
||||
seq_lens=torch.tensor([rows], device="cuda", dtype=torch.int32),
|
||||
out_cache_loc=torch.arange(rows, device="cuda"),
|
||||
seq_lens_sum=rows,
|
||||
num_token_non_padded=live,
|
||||
)
|
||||
output = torch.full_like(x, float("nan"))
|
||||
finite_logits = logits.clone()
|
||||
hook_weights = []
|
||||
if runner_kind != "fused":
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.marlin import MarlinMoeQuantInfo
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
|
||||
config = MoeRunnerConfig(
|
||||
num_experts=experts,
|
||||
num_local_experts=experts,
|
||||
hidden_size=hidden,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
top_k=topk,
|
||||
params_dtype=x.dtype,
|
||||
runner_backend=MoeRunnerBackend.MARLIN,
|
||||
inplace=False,
|
||||
)
|
||||
quant_info = MarlinMoeQuantInfo(
|
||||
w13_qweight=w1,
|
||||
w2_qweight=w2,
|
||||
w13_scales=s1,
|
||||
w2_scales=s2,
|
||||
w13_g_idx_sort_indices=p1,
|
||||
w2_g_idx_sort_indices=p2,
|
||||
weight_bits=num_bits,
|
||||
w13_g_idx=g1,
|
||||
w2_g_idx=g2,
|
||||
w13_qzeros=z1,
|
||||
w2_qzeros=z2,
|
||||
)
|
||||
if runner_kind == "stock_lora":
|
||||
from sglang.srt.lora.lora_moe_runner_marlin import (
|
||||
MarlinLoraRunnerCore,
|
||||
)
|
||||
from sglang.srt.lora.lora_moe_runners import LoRAHooks
|
||||
|
||||
core = MarlinLoraRunnerCore(config)
|
||||
|
||||
def after_down(activation, projected, weights, ids):
|
||||
hook_weights[:] = [weights, ids]
|
||||
|
||||
hooks = LoRAHooks(after_down=after_down)
|
||||
else:
|
||||
from sglang.srt.lora.marlin_lora_temp.moe_runner import (
|
||||
_use_shared_outer_factorization,
|
||||
fused_experts_experimental_sgl_marlin_lora,
|
||||
)
|
||||
|
||||
# A zero adapter must preserve base output, including NaN-padded
|
||||
# routing weights consumed by the factored rank-sum reduction.
|
||||
rank = 8
|
||||
lora_info = SimpleNamespace(
|
||||
lora_use_virtual_experts=True,
|
||||
max_lora_rank=rank,
|
||||
has_active_lora=True,
|
||||
experts_shared_outer_loras=True,
|
||||
gate_up_lora_a_weights=x.new_zeros((1, 1, 2 * rank, hidden)),
|
||||
gate_up_lora_b_weights=x.new_zeros(
|
||||
(1, experts, 2 * intermediate, rank)
|
||||
),
|
||||
down_lora_a_weights=x.new_zeros(
|
||||
(1, experts, rank, intermediate)
|
||||
),
|
||||
down_lora_b_weights=x.new_zeros((1, 1, hidden, rank)),
|
||||
token_lora_mapping=torch.zeros(
|
||||
rows, device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
assert _use_shared_outer_factorization(lora_info, rows, topk)
|
||||
|
||||
def run():
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=None), forward_batch=batch
|
||||
):
|
||||
routing = router(x, logits)
|
||||
if runner_kind != "fused":
|
||||
dispatch = StandardDispatchOutput(x, None, routing)
|
||||
if runner_kind == "stock_lora":
|
||||
return core.run_from_dispatch(
|
||||
dispatch, quant_info, config, hooks
|
||||
).hidden_states
|
||||
return fused_experts_experimental_sgl_marlin_lora(
|
||||
dispatch, quant_info, config, lora_info
|
||||
).hidden_states
|
||||
return fused_marlin_moe(
|
||||
x,
|
||||
w1,
|
||||
w2,
|
||||
s1,
|
||||
s2,
|
||||
logits,
|
||||
routing.topk_weights,
|
||||
routing.topk_ids,
|
||||
g_idx1=g1,
|
||||
g_idx2=g2,
|
||||
sort_indices1=p1,
|
||||
sort_indices2=p2,
|
||||
w1_zeros=z1,
|
||||
w2_zeros=z2,
|
||||
num_bits=num_bits,
|
||||
)
|
||||
|
||||
weights, ids = torch.topk(torch.softmax(logits, dim=-1), topk)
|
||||
weights /= weights.sum(dim=-1, keepdim=True)
|
||||
gate_up = torch.einsum("mh,mtih->mti", x, w1_ref[ids])
|
||||
gate, up = gate_up.chunk(2, dim=-1)
|
||||
activated = torch.nn.functional.silu(gate) * up
|
||||
projected = torch.einsum("mti,mthi->mth", activated, w2_ref[ids])
|
||||
reference = (projected.float() * weights[:, :, None]).sum(1).to(x.dtype)
|
||||
with patch.object(zero_copy_context, "get_moe_output", return_value=output):
|
||||
for _ in range(3):
|
||||
run()
|
||||
torch.cuda.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = run()
|
||||
for count in (max(rows - 1, 0), rows, 0, min(rows, 2)):
|
||||
logits.copy_(finite_logits)
|
||||
logits[count:].fill_(float("nan"))
|
||||
live.fill_(count)
|
||||
actual.fill_(float("nan"))
|
||||
graph.replay()
|
||||
expected = reference.clone()
|
||||
expected[count:].zero_()
|
||||
torch.testing.assert_close(actual, expected, rtol=0.03, atol=0.02)
|
||||
assert torch.isfinite(actual).all()
|
||||
assert torch.count_nonzero(actual[count:]) == 0
|
||||
if hook_weights:
|
||||
weights, ids = hook_weights
|
||||
assert torch.count_nonzero(weights[ids < 0]) == 0
|
||||
|
||||
|
||||
def test_moe_wna16_marlin_inactive_output_ownership():
|
||||
"""An allocated output zeros inactive rows; caller-owned rows stay untouched."""
|
||||
torch.manual_seed(57)
|
||||
m, e, k, n, topk, block_m = 1, 8, 256, 256, 2, 8
|
||||
_, qweight, scales, zeros, g_idx, permutation = _marlin_inactive_weights(
|
||||
e, n, k, scalar_types.uint4b8
|
||||
)
|
||||
x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16)
|
||||
ids = torch.full((m, topk), -1, device="cuda", dtype=torch.int32)
|
||||
routing_weights = torch.ones((m, topk), device="cuda", dtype=torch.float32)
|
||||
sorted_ids, expert_ids, num_post = moe_align_block_size(ids, block_m, e)
|
||||
workspace = torch.zeros(
|
||||
torch.cuda.get_device_properties(x.device).multi_processor_count * 4,
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
caller_output = torch.full((m * topk, n), 3.0, device="cuda", dtype=x.dtype)
|
||||
original_empty = torch.empty
|
||||
|
||||
def poisoned_empty(*args, **kwargs):
|
||||
tensor = original_empty(*args, **kwargs)
|
||||
if tensor.is_floating_point():
|
||||
tensor.fill_(float("nan"))
|
||||
return tensor
|
||||
|
||||
for output in (None, caller_output):
|
||||
with patch.object(torch, "empty", side_effect=poisoned_empty):
|
||||
result = _run_single_gemm(
|
||||
moe_wna16_marlin_gemm,
|
||||
x,
|
||||
output,
|
||||
qweight,
|
||||
scales,
|
||||
zeros,
|
||||
g_idx,
|
||||
permutation,
|
||||
workspace,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_post,
|
||||
routing_weights,
|
||||
scalar_types.uint4b8,
|
||||
block_m,
|
||||
topk,
|
||||
m,
|
||||
n,
|
||||
k,
|
||||
False,
|
||||
True,
|
||||
True,
|
||||
)
|
||||
expected = (
|
||||
torch.zeros_like(result) if output is None else torch.full_like(result, 3.0)
|
||||
)
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_marlin_lora_direct_inactive_routes():
|
||||
"""Direct LoRA projections overwrite inactive routes with finite zeros."""
|
||||
from sglang.srt.lora.marlin_lora_temp.direct_decode import (
|
||||
direct_decode_down_shrink,
|
||||
direct_decode_gate_expand,
|
||||
)
|
||||
|
||||
torch.manual_seed(51)
|
||||
rows, topk, experts, intermediate, rank = 3, 6, 8, 384, 32
|
||||
ids = torch.tensor(
|
||||
[[0, 1, 2, 3, 4, 5], [-1, -1, -1, -1, -1, -1], [2, -1, 4, 5, -1, 7]],
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
slots = torch.zeros(rows, device="cuda", dtype=torch.int32)
|
||||
shared = torch.randn((rows, 2 * rank), device="cuda", dtype=torch.bfloat16)
|
||||
gate_b = (
|
||||
torch.randn(
|
||||
(1, experts, 2 * intermediate, rank), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
* 0.05
|
||||
)
|
||||
down_a = (
|
||||
torch.randn(
|
||||
(1, experts, rank, intermediate), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
* 0.05
|
||||
)
|
||||
activation = torch.randn(
|
||||
(rows * topk, intermediate), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
activation.view(rows, topk, intermediate)[ids < 0] = float("nan")
|
||||
gate_output = torch.full(
|
||||
(rows, topk, 2 * intermediate),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
down_output = torch.full(
|
||||
(rows, topk, rank), float("nan"), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
direct_decode_gate_expand(shared, gate_b, ids, slots, gate_output)
|
||||
direct_decode_down_shrink(activation, down_a, ids, slots, down_output)
|
||||
expected_gate = torch.zeros_like(gate_output)
|
||||
expected_down = torch.zeros_like(down_output)
|
||||
for token in range(rows):
|
||||
for route in range(topk):
|
||||
expert = int(ids[token, route])
|
||||
if expert < 0:
|
||||
continue
|
||||
for half in range(2):
|
||||
start = half * intermediate
|
||||
expected_gate[token, route, start : start + intermediate] = (
|
||||
gate_b[0, expert, start : start + intermediate].float()
|
||||
@ shared[token, half * rank : (half + 1) * rank].float()
|
||||
).to(gate_output.dtype)
|
||||
expected_down[token, route] = (
|
||||
down_a[0, expert].float() @ activation[token * topk + route].float()
|
||||
).to(down_output.dtype)
|
||||
torch.testing.assert_close(gate_output, expected_gate, rtol=0.03, atol=0.01)
|
||||
torch.testing.assert_close(down_output, expected_down, rtol=0.03, atol=0.01)
|
||||
assert torch.count_nonzero(gate_output[ids < 0]) == 0
|
||||
assert torch.count_nonzero(down_output[ids < 0]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fused_norm", [False, True])
|
||||
def test_muse_marlin_inactive_routes(fused_norm):
|
||||
"""Muse maps global routes once and masks inactive weights before either norm tail."""
|
||||
from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts
|
||||
|
||||
torch.manual_seed(67)
|
||||
rows, local_experts, hidden, intermediate, topk = 4, 2, 256, 256, 2
|
||||
dtype = torch.bfloat16
|
||||
layer = MuseSpark1xFp4Experts.__new__(MuseSpark1xFp4Experts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.quant_config = SimpleNamespace(group_size=16)
|
||||
layer.moe_runner_config = SimpleNamespace(is_gated=True)
|
||||
layer.params_dtype = dtype
|
||||
layer.intermediate_size_per_partition = intermediate
|
||||
layer.num_local_experts = local_experts
|
||||
layer.global_num_experts = 4
|
||||
layer._expert_map = torch.tensor([-1, -1, 0, 1], device="cuda", dtype=torch.int32)
|
||||
references = []
|
||||
for prefix, n, k in (
|
||||
("w13", 2 * intermediate, hidden),
|
||||
("w2", hidden, intermediate),
|
||||
):
|
||||
packed, scales, global_scales, refs = zip(
|
||||
*[make_nvfp4_weight_and_ref(n, k, dtype) for _ in range(local_experts)]
|
||||
)
|
||||
for suffix, tensors in (
|
||||
("weight", packed),
|
||||
("weight_scale", scales),
|
||||
("weight_scale_2", global_scales),
|
||||
):
|
||||
setattr(
|
||||
layer,
|
||||
f"{prefix}_{suffix}",
|
||||
torch.nn.Parameter(torch.stack(tensors), requires_grad=False),
|
||||
)
|
||||
references.append(torch.stack(refs))
|
||||
prepare_moe_nvfp4_layer_for_marlin(layer)
|
||||
x = torch.randn((rows, hidden), device="cuda", dtype=dtype) / 20
|
||||
ids = torch.tensor(
|
||||
[[2, 3], [3, -1], [0, 2], [-1, -1]], device="cuda", dtype=torch.int32
|
||||
)
|
||||
local_ids = torch.tensor(
|
||||
[[0, 1], [1, -1], [-1, 0], [-1, -1]], device="cuda", dtype=torch.int32
|
||||
)
|
||||
weights = torch.tensor(
|
||||
[[0.25, 0.75], [0.5, 0.5], [0.25, 0.75], [0.5, 0.5]], device="cuda"
|
||||
)
|
||||
weights[local_ids < 0] = float("nan")
|
||||
norm = torch.nn.RMSNorm(hidden, eps=1e-6, device="cuda", dtype=dtype)
|
||||
layer._post_expert_norm = norm
|
||||
layer._post_expert_norm_weight = norm.weight.detach() if fused_norm else None
|
||||
layer._post_expert_norm_eps = norm.eps
|
||||
layer._post_expert_norm_weight_first = False
|
||||
|
||||
expert_output, mapped = layer._marlin_expert_outputs(x, weights, ids)
|
||||
torch.testing.assert_close(mapped, local_ids, rtol=0, atol=0)
|
||||
assert torch.count_nonzero(expert_output[local_ids < 0]) == 0
|
||||
expected_experts = torch.zeros((rows, topk, hidden), device="cuda", dtype=dtype)
|
||||
w13_ref, w2_ref = references
|
||||
for row in range(rows):
|
||||
for route in range(topk):
|
||||
expert = int(local_ids[row, route])
|
||||
if expert < 0:
|
||||
continue
|
||||
gate_up = x[row] @ w13_ref[expert].T
|
||||
gate, up = gate_up.chunk(2)
|
||||
activated = torch.nn.functional.silu(gate) * up
|
||||
expected_experts[row, route] = activated @ w2_ref[expert].T
|
||||
torch.testing.assert_close(expert_output, expected_experts, rtol=0.05, atol=0.25)
|
||||
safe_weights = weights.masked_fill(local_ids < 0, 0.0)
|
||||
expected = (
|
||||
(norm(expected_experts).float() * safe_weights[..., None]).sum(1).to(dtype)
|
||||
)
|
||||
actual = layer._forward_marlin(x, weights, ids, norm)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.05, atol=0.25)
|
||||
assert torch.isfinite(actual).all()
|
||||
assert torch.count_nonzero(actual[-1]) == 0
|
||||
assert torch.isnan(weights[local_ids < 0]).all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v", "-s"]))
|
||||
|
||||
@@ -59,7 +59,26 @@ class TestSelectExpertsPerRankSharedSlots(CustomTestCase):
|
||||
"has_per_rank_fused_shared_slots",
|
||||
side_effect=lambda num_fused: num_fused > 0,
|
||||
),
|
||||
get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK),
|
||||
# Model one rank of TP=EP; routing itself issues no collectives.
|
||||
get_parallel().override(
|
||||
tp_size=self.EP_SIZE,
|
||||
tp_rank=self.EP_RANK,
|
||||
tp_group=None,
|
||||
attn_tp_size=self.EP_SIZE,
|
||||
attn_tp_rank=self.EP_RANK,
|
||||
attn_tp_group=None,
|
||||
attn_dp_size=1,
|
||||
attn_dp_rank=0,
|
||||
attn_cp_size=1,
|
||||
attn_cp_rank=0,
|
||||
moe_ep_size=self.EP_SIZE,
|
||||
moe_ep_rank=self.EP_RANK,
|
||||
moe_ep_group=None,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_dp_size=1,
|
||||
moe_dp_rank=0,
|
||||
),
|
||||
):
|
||||
out = select_experts(
|
||||
torch.empty(logits.shape[0], 16, device=logits.device),
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Padded routes must contribute zero even when every expert is local."""
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.moe.fused_moe_triton_kernels import invoke_fused_moe_kernel
|
||||
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.triton import (
|
||||
TritonMoeQuantInfo,
|
||||
TritonRunnerCore,
|
||||
pre_permute_standard_to_triton,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend, initialize_moe_config
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=20, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestTritonMoePadding(CustomTestCase):
|
||||
ROWS, EXPERTS, TOP_K, HIDDEN, INTERMEDIATE = 4, 4, 2, 128, 128
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.enterContext(
|
||||
get_context().override_server_args(
|
||||
model_path="dummy",
|
||||
device="cuda",
|
||||
moe_runner_backend="triton",
|
||||
ep_dispatch_algorithm=None,
|
||||
enable_eplb=False,
|
||||
enable_waterfill=False,
|
||||
)
|
||||
)
|
||||
moe_flags = get_flags().moe
|
||||
scoped_flags = {
|
||||
name: getattr(moe_flags, name) for name in moe_flags.__struct_fields__
|
||||
}
|
||||
scoped_flags["in_speculative_scope"] = False
|
||||
self.enterContext(moe_flags.override(**scoped_flags))
|
||||
initialize_moe_config()
|
||||
self.enterContext(
|
||||
get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_dp_size=1,
|
||||
attn_cp_size=1,
|
||||
tp_group=None,
|
||||
)
|
||||
)
|
||||
torch.manual_seed(123)
|
||||
|
||||
@staticmethod
|
||||
def _expert_view(shape, dtype, *, quantized=False):
|
||||
# A normal view with an extra leading record makes selecting an inactive
|
||||
# expert numerically observable. Only actual experts contain finite data.
|
||||
if quantized:
|
||||
storage = torch.randint(
|
||||
120, 137, (shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype
|
||||
)
|
||||
else:
|
||||
storage = (
|
||||
torch.randn((shape[0] + 1, *shape[1:]), device="cuda", dtype=dtype)
|
||||
* 0.05
|
||||
)
|
||||
storage[0].fill_(float("nan"))
|
||||
return storage[1:]
|
||||
|
||||
def _fixture(self, dtype, quantized=False):
|
||||
x = torch.randn((self.ROWS, self.HIDDEN), device="cuda", dtype=dtype) * 0.1
|
||||
logits = torch.tensor(
|
||||
[
|
||||
[9.0, 8.0, 1.0, 0.0],
|
||||
[0.0, 9.0, 8.0, 1.0],
|
||||
[1.0, 0.0, 9.0, 8.0],
|
||||
[8.0, 1.0, 0.0, 9.0],
|
||||
],
|
||||
device="cuda",
|
||||
)
|
||||
shapes = [
|
||||
(self.EXPERTS, 2 * self.INTERMEDIATE, self.HIDDEN),
|
||||
(self.EXPERTS, self.HIDDEN, self.INTERMEDIATE),
|
||||
]
|
||||
if quantized:
|
||||
weights = [
|
||||
self._expert_view(shape, torch.uint8, quantized=True)
|
||||
for shape in shapes
|
||||
]
|
||||
scales = []
|
||||
for shape in shapes:
|
||||
storage = torch.full(
|
||||
(self.EXPERTS + 1, shape[1], shape[2] // 32),
|
||||
0.01,
|
||||
device="cuda",
|
||||
dtype=dtype,
|
||||
)
|
||||
storage[0].fill_(float("nan"))
|
||||
scales.append(storage[1:])
|
||||
quant = TritonMoeQuantInfo(
|
||||
*weights,
|
||||
use_int8_w8a16=True,
|
||||
w13_scale=scales[0],
|
||||
w2_scale=scales[1],
|
||||
block_shape=[0, 32],
|
||||
)
|
||||
dense_weights = [
|
||||
(weight.to(dtype) - 128) * scale.repeat_interleave(32, dim=-1)
|
||||
for weight, scale in zip(weights, scales)
|
||||
]
|
||||
else:
|
||||
weights = [self._expert_view(shape, dtype) for shape in shapes]
|
||||
quant = TritonMoeQuantInfo(*weights)
|
||||
dense_weights = weights
|
||||
router = TopK(
|
||||
top_k=self.TOP_K,
|
||||
layer_id=0,
|
||||
output_format=TopKOutputFormat.STANDARD,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
router.bind_runner_backend(MoeRunnerBackend.TRITON)
|
||||
batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
batch_size=self.ROWS,
|
||||
input_ids=torch.arange(self.ROWS, device="cuda"),
|
||||
req_pool_indices=torch.arange(self.ROWS, device="cuda"),
|
||||
seq_lens=torch.ones(self.ROWS, device="cuda", dtype=torch.int32),
|
||||
out_cache_loc=torch.arange(self.ROWS, device="cuda"),
|
||||
seq_lens_sum=self.ROWS,
|
||||
num_token_non_padded=torch.tensor(2, device="cuda", dtype=torch.int32),
|
||||
)
|
||||
config = MoeRunnerConfig(
|
||||
num_experts=self.EXPERTS,
|
||||
num_local_experts=self.EXPERTS,
|
||||
hidden_size=self.HIDDEN,
|
||||
intermediate_size_per_partition=self.INTERMEDIATE,
|
||||
top_k=self.TOP_K,
|
||||
num_fused_shared_experts=0,
|
||||
params_dtype=dtype,
|
||||
runner_backend=MoeRunnerBackend.TRITON,
|
||||
inplace=False,
|
||||
)
|
||||
return x, logits, quant, dense_weights, router, batch, config
|
||||
|
||||
@staticmethod
|
||||
def _prepare(x, routes, quant, config):
|
||||
state = {}
|
||||
prepared = pre_permute_standard_to_triton(
|
||||
StandardDispatchOutput(x, None, routes), quant, config, state
|
||||
)
|
||||
return prepared, state
|
||||
|
||||
def _run(self, x, routes, quant, config):
|
||||
prepared, state = self._prepare(x, routes, quant, config)
|
||||
return TritonRunnerCore(config).run(prepared, quant, state).hidden_states
|
||||
|
||||
def _dense_reference(self, x, logits, weights):
|
||||
probabilities = torch.softmax(logits, dim=-1)
|
||||
scores, ids = probabilities.topk(self.TOP_K, dim=-1)
|
||||
scores /= scores.sum(dim=-1, keepdim=True)
|
||||
result = torch.zeros_like(x)
|
||||
for row in range(x.shape[0]):
|
||||
for slot in range(self.TOP_K):
|
||||
expert = int(ids[row, slot])
|
||||
gate, up = F.linear(x[row], weights[0][expert]).chunk(2)
|
||||
activated = (F.silu(gate.float()) * up.float()).to(x.dtype)
|
||||
result[row] += (
|
||||
F.linear(activated, weights[1][expert]).float() * scores[row, slot]
|
||||
).to(x.dtype)
|
||||
return result
|
||||
|
||||
def test_runner_observes_updated_batch_bound(self):
|
||||
"""An inactive row must stay finite and zero through both expert GEMMs."""
|
||||
for dtype, quantized in [
|
||||
(torch.bfloat16, False),
|
||||
(torch.float16, False),
|
||||
(torch.bfloat16, True),
|
||||
]:
|
||||
with self.subTest(dtype=dtype, quantized=quantized):
|
||||
x, logits, quant, weights, router, batch, config = self._fixture(
|
||||
dtype, quantized
|
||||
)
|
||||
dense = self._dense_reference(x, logits, weights)
|
||||
# Reuse the batch and its device scalar, as static graph buffers do.
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=None), forward_batch=batch
|
||||
):
|
||||
for live in (2, 3, 0, self.ROWS):
|
||||
with self.subTest(live=live):
|
||||
batch.num_token_non_padded.fill_(live)
|
||||
routes = router.forward_cuda(x, logits)
|
||||
self.assertTrue(torch.all(routes.topk_ids[live:] == -1))
|
||||
output = self._run(x, routes, quant, config)
|
||||
self.assertTrue(torch.isfinite(output).all())
|
||||
torch.testing.assert_close(
|
||||
output[live:],
|
||||
torch.zeros_like(output[live:]),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[:live], dense[:live], rtol=0.04, atol=2e-4
|
||||
)
|
||||
if live:
|
||||
# A cropped input has no inactive routes at all.
|
||||
cropped_routes = router.forward_cuda(
|
||||
x[:live], logits[:live]
|
||||
)
|
||||
cropped = self._run(
|
||||
x[:live], cropped_routes, quant, config
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[:live], cropped, rtol=0.01, atol=2e-5
|
||||
)
|
||||
|
||||
def test_inactive_routes_preserve_output_layouts(self):
|
||||
"""Skipping a route must respect sorted, accumulated and half-width outputs."""
|
||||
x, logits, quant, _, router, batch, config = self._fixture(torch.bfloat16)
|
||||
with forward_context(ForwardContext(attn_backend=None), forward_batch=batch):
|
||||
routes = router.forward_cuda(x, logits)
|
||||
prepared, state = self._prepare(x, routes, quant, config)
|
||||
count = int(prepared.num_tokens_post_padded)
|
||||
block = state["config"]["BLOCK_SIZE_M"]
|
||||
sorted_ids = prepared.sorted_token_ids[:count].tolist()
|
||||
expert_ids = prepared.expert_ids[: (count + block - 1) // block].tolist()
|
||||
for layout in ("sorted", "add", "sum", "swiglu"):
|
||||
with self.subTest(layout=layout):
|
||||
width = (
|
||||
self.INTERMEDIATE if layout == "swiglu" else 2 * self.INTERMEDIATE
|
||||
)
|
||||
rows = (
|
||||
count
|
||||
if layout == "sorted"
|
||||
else self.ROWS
|
||||
if layout == "sum"
|
||||
else self.ROWS * self.TOP_K
|
||||
)
|
||||
output = torch.full((rows, width), 0.375, device="cuda", dtype=x.dtype)
|
||||
expected = output.clone()
|
||||
for position, route in enumerate(sorted_ids):
|
||||
expert = expert_ids[position // block]
|
||||
if expert == -1 or route >= self.ROWS * self.TOP_K:
|
||||
continue
|
||||
value = F.linear(x[route // self.TOP_K], quant.w13_weight[expert])
|
||||
if layout == "swiglu":
|
||||
gate, up = value[::2], value[1::2]
|
||||
value = (F.silu(gate.float()) * up.float()).to(x.dtype)
|
||||
target = (
|
||||
position
|
||||
if layout == "sorted"
|
||||
else route // self.TOP_K
|
||||
if layout == "sum"
|
||||
else route
|
||||
)
|
||||
if layout in ("add", "sum"):
|
||||
expected[target] += value
|
||||
else:
|
||||
expected[target] = value
|
||||
invoke_fused_moe_kernel(
|
||||
x,
|
||||
quant.w13_weight,
|
||||
None,
|
||||
output,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
routes.topk_weights,
|
||||
routes.topk_ids,
|
||||
prepared.sorted_token_ids,
|
||||
prepared.expert_ids,
|
||||
prepared.num_tokens_post_padded,
|
||||
False,
|
||||
self.TOP_K,
|
||||
state["config"],
|
||||
compute_type=tl.bfloat16,
|
||||
use_fp8_w8a8=False,
|
||||
use_int8_w8a8=False,
|
||||
use_int8_w8a16=False,
|
||||
use_int4_w4a16=False,
|
||||
per_channel_quant=False,
|
||||
filter_expert=False,
|
||||
c_sorted=layout == "sorted",
|
||||
fuse_add_to_output=layout == "add",
|
||||
add_output_mask=torch.ones(
|
||||
self.ROWS, device="cuda", dtype=torch.bool
|
||||
),
|
||||
fuse_sum_all_reduce=layout == "sum",
|
||||
router_topk=self.TOP_K,
|
||||
fuse_swiglu=layout == "swiglu",
|
||||
)
|
||||
self.assertTrue(torch.isfinite(output).all())
|
||||
torch.testing.assert_close(output, expected, rtol=0.02, atol=2e-3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferDispatcher,
|
||||
_WorkspaceNamespace,
|
||||
@@ -126,10 +127,20 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
router_logits = torch.tensor(
|
||||
[[0.1, 3.0, 0.2, 2.0], [5.0, 0.0, 4.0, 0.1], [0.0, 0.1, 0.2, 9.0]]
|
||||
)
|
||||
logical_to_physical = torch.tensor([2, 0, 3, 1])
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=TopKConfig(top_k=2, torch_native=True),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo(
|
||||
ep_dispatch_algorithm="static",
|
||||
partial_logical_to_rank_dispatch_physical_map=logical_to_physical,
|
||||
partial_logical_to_all_physical_map=logical_to_physical[:, None],
|
||||
partial_logical_to_all_physical_map_num_valid=torch.ones(
|
||||
4, dtype=torch.int64
|
||||
),
|
||||
num_physical_experts=4,
|
||||
),
|
||||
)
|
||||
|
||||
_dispatch_without_dp(
|
||||
@@ -142,7 +153,7 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
expert_ids = moe_a2a.token_selected_experts
|
||||
self.assertEqual(expert_ids.dtype, torch.int32)
|
||||
self.assertEqual(
|
||||
expert_ids.sort(dim=-1).values.tolist(), [[1, 3], [0, 2], [2, 3]]
|
||||
expert_ids.sort(dim=-1).values.tolist(), [[0, 1], [2, 3], [1, 3]]
|
||||
)
|
||||
topk_weights = moe_a2a.payloads[moe_a2a.expert_id_payload_index + 1]
|
||||
self.assertEqual(topk_weights.shape, (3, 2))
|
||||
|
||||
@@ -531,13 +531,13 @@ class TestFusedSharedExpertScaling(CustomTestCase):
|
||||
def test_glm53_idle_rank_preserves_routed_topk_contract(self):
|
||||
"""An idle rank must return the pre-append top-8 shape; returning top-9
|
||||
here would make the AITER postprocessor append shared ID 288 twice."""
|
||||
topk = TopK.__new__(TopK)
|
||||
torch.nn.Module.__init__(topk)
|
||||
topk.topk_config = TopKConfig(
|
||||
top_k=9, shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
|
||||
)
|
||||
topk.enable_waterfill = False
|
||||
topk.waterfill_balancer = None
|
||||
with (
|
||||
get_context().override_server_args(enable_waterfill=False),
|
||||
get_model().override(
|
||||
shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
|
||||
),
|
||||
):
|
||||
topk = TopK(top_k=8, layer_id=0)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
|
||||
@@ -8,6 +8,7 @@ collectives and graph replay are out of scope.
|
||||
|
||||
import sys
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -61,11 +62,6 @@ def runtime(monkeypatch):
|
||||
monkeypatch.setattr(mega, "get_dp_global_num_tokens", lambda: state.global_tokens)
|
||||
monkeypatch.setattr(mega, "_ensure_mori_shmem", lambda: None)
|
||||
monkeypatch.setattr(mega, "_ep_rank_world", lambda: (state.rank, 8))
|
||||
monkeypatch.setattr(
|
||||
mega,
|
||||
"ExpertLocationDispatchInfo",
|
||||
SimpleNamespace(init_new=lambda layer_id: None),
|
||||
)
|
||||
utils = ModuleType("sglang.srt.models.deepseek_common.utils")
|
||||
utils._use_aiter = True
|
||||
monkeypatch.setitem(sys.modules, utils.__name__, utils)
|
||||
@@ -100,6 +96,13 @@ def runtime(monkeypatch):
|
||||
state.extend = extend
|
||||
state.capture = capture
|
||||
state.global_tokens = global_tokens
|
||||
topk = Mock(
|
||||
return_value=SimpleNamespace(
|
||||
topk_ids=torch.zeros(local_rows, TOPK, dtype=torch.int64),
|
||||
topk_weights=torch.ones(local_rows, TOPK),
|
||||
)
|
||||
)
|
||||
topk.routing_inputs.return_value = (None, None)
|
||||
moe = SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
hidden_size=HIDDEN,
|
||||
@@ -110,10 +113,7 @@ def runtime(monkeypatch):
|
||||
num_fused_shared_experts=0,
|
||||
experts=state.layer if layer is None else layer,
|
||||
gate=lambda h, forward_batch=None: torch.zeros(h.shape[0], EXPERTS),
|
||||
topk=lambda h, logits, **kwargs: SimpleNamespace(
|
||||
topk_ids=torch.zeros(h.shape[0], TOPK, dtype=torch.int64),
|
||||
topk_weights=torch.ones(h.shape[0], TOPK),
|
||||
),
|
||||
topk=topk,
|
||||
is_hash=False,
|
||||
layer_id=0,
|
||||
routed_scaling_factor=1.0,
|
||||
|
||||
@@ -17,10 +17,17 @@ maybe_stub_sgl_kernel()
|
||||
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
|
||||
from sglang.srt.layers.moe.topk import TopKConfig
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import draft_model_build_scope
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
from sglang.srt.runtime_context import get_context, get_exec, get_flags, get_parallel
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.runtime_context import (
|
||||
get_context,
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_model,
|
||||
get_parallel,
|
||||
)
|
||||
|
||||
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
|
||||
|
||||
@@ -257,11 +264,6 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
),
|
||||
# Toy shapes.
|
||||
patch.object(mega_moe, "check_mega_moe_shapes"),
|
||||
patch.object(
|
||||
mega_moe.ExpertLocationDispatchInfo,
|
||||
"init_new",
|
||||
return_value=object(),
|
||||
),
|
||||
get_parallel().override(
|
||||
moe_ep_group=SimpleNamespace(device_group=object())
|
||||
),
|
||||
@@ -356,6 +358,14 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
expert boundary exposes accidentally routing image rows as text, and
|
||||
padded rows must contribute nothing.
|
||||
"""
|
||||
config = get_context().override_server_args(
|
||||
model_path="dummy",
|
||||
enable_waterfill=False,
|
||||
ep_dispatch_algorithm=None,
|
||||
enable_eplb=False,
|
||||
)
|
||||
config.install()
|
||||
self.addCleanup(config.restore)
|
||||
for fused_shared in (0, 1):
|
||||
for scale_in_topk in (False, True):
|
||||
with self.subTest(
|
||||
@@ -383,12 +393,17 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
|
||||
with (
|
||||
get_flags().moe.override(a2a_backend=MoeA2ABackend.MEGAMOE),
|
||||
get_parallel().override(moe_ep_size=2, moe_ep_rank=1),
|
||||
patch.object(
|
||||
mega_moe.ExpertLocationDispatchInfo,
|
||||
"init_new",
|
||||
return_value=None,
|
||||
get_parallel().override(
|
||||
tp_size=2,
|
||||
tp_rank=1,
|
||||
attn_tp_size=2,
|
||||
attn_tp_rank=1,
|
||||
moe_ep_size=2,
|
||||
moe_ep_rank=1,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
),
|
||||
forward_context(ForwardContext(None), forward_batch=batch),
|
||||
patch.object(mega_moe, "run_mega_routed_experts", run_experts),
|
||||
# The fused padded-row fill requires Triton/GPU.
|
||||
patch(
|
||||
@@ -437,42 +452,46 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
|
||||
@staticmethod
|
||||
def _v41_moe(fused_shared, scale_in_topk):
|
||||
return SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
hidden_size=4,
|
||||
num_experts_per_tok=2,
|
||||
moe_intermediate_size=8,
|
||||
image_token_id=99,
|
||||
with (
|
||||
get_context().override_server_args(),
|
||||
get_model().override(
|
||||
shared_experts=SharedExpertsConfig(
|
||||
num_fused_shared_experts=fused_shared
|
||||
)
|
||||
),
|
||||
experts=SimpleNamespace(
|
||||
should_fuse_routed_scaling_factor_in_topk=scale_in_topk,
|
||||
moe_runner_config=SimpleNamespace(swiglu_limit=None),
|
||||
uses_expert_location=False,
|
||||
),
|
||||
gate=MagicMock(
|
||||
return_value=torch.zeros((3, 4)),
|
||||
e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]),
|
||||
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]),
|
||||
),
|
||||
topk=SimpleNamespace(
|
||||
topk_config=TopKConfig(
|
||||
top_k=2 + fused_shared,
|
||||
shared_experts=SharedExpertsConfig(
|
||||
num_fused_shared_experts=fused_shared
|
||||
),
|
||||
):
|
||||
return SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
hidden_size=4,
|
||||
num_experts_per_tok=2,
|
||||
moe_intermediate_size=8,
|
||||
image_token_id=99,
|
||||
),
|
||||
experts=SimpleNamespace(
|
||||
should_fuse_routed_scaling_factor_in_topk=scale_in_topk,
|
||||
moe_runner_config=SimpleNamespace(swiglu_limit=None),
|
||||
),
|
||||
gate=MagicMock(
|
||||
return_value=torch.zeros((3, 4)),
|
||||
e_score_correction_bias=torch.tensor([4.0, 3.0, 0.0, 0.0]),
|
||||
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 3.0, 4.0]),
|
||||
),
|
||||
topk=TopK(
|
||||
top_k=2,
|
||||
layer_id=0,
|
||||
scoring_func="sqrtsoftplus",
|
||||
renormalize=True,
|
||||
routed_scaling_factor=2.0,
|
||||
apply_routed_scaling_factor_on_output=scale_in_topk,
|
||||
)
|
||||
),
|
||||
is_hash=False,
|
||||
num_fused_shared_experts=fused_shared,
|
||||
layer_id=0,
|
||||
routed_scaling_factor=2.0,
|
||||
routed_scaling_factor_fused_in_topk=scale_in_topk,
|
||||
mega_shared_l1_weights=None,
|
||||
mega_shared_l2_weights=None,
|
||||
)
|
||||
),
|
||||
is_hash=False,
|
||||
num_fused_shared_experts=fused_shared,
|
||||
layer_id=0,
|
||||
routed_scaling_factor=2.0,
|
||||
routed_scaling_factor_fused_in_topk=scale_in_topk,
|
||||
mega_shared_l1_weights=None,
|
||||
mega_shared_l2_weights=None,
|
||||
)
|
||||
|
||||
def test_shape_check_uses_pinned_fp8_fp4_alignment(self):
|
||||
# Packed SF groups need 128 elements, not a 16-byte token SF row.
|
||||
|
||||
@@ -0,0 +1,724 @@
|
||||
"""Routing follows the owning layer and active batch across target/draft and TBO."""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.batch_overlap.operations import (
|
||||
ExecutionOperation,
|
||||
_StageExecutor,
|
||||
)
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb import expert_location_dispatch
|
||||
from sglang.srt.eplb.expert_location import ExpertLocationMetadata
|
||||
from sglang.srt.hardware_backend.npu.moe.topk import fused_topk_npu
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode
|
||||
from sglang.srt.layers.moe import hash_topk as hash_topk_module
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.hash_topk import HashTopK
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutputPacked,
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
fused_topk_torch_native,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
MoeRunnerBackend,
|
||||
draft_model_build_scope,
|
||||
initialize_moe_config,
|
||||
)
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.multimodal.dsv41.vl_routing import vision_topk
|
||||
from sglang.srt.runtime_context import (
|
||||
get_context,
|
||||
get_exec,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_resources,
|
||||
)
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftInput
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def _batch(real):
|
||||
empty = torch.empty(0, dtype=torch.int32)
|
||||
batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
batch_size=4,
|
||||
input_ids=torch.arange(4),
|
||||
req_pool_indices=empty,
|
||||
seq_lens=empty,
|
||||
out_cache_loc=empty,
|
||||
seq_lens_sum=0,
|
||||
)
|
||||
batch.num_token_non_padded = torch.tensor(real, dtype=torch.int32)
|
||||
batch.global_num_token_non_padded = batch.num_token_non_padded
|
||||
batch.dp_padding_mode = DpPaddingMode.SUM_LEN
|
||||
batch.global_dp_buffer_len = batch.tbo_padded_len = 4
|
||||
return batch
|
||||
|
||||
|
||||
def _placement(num_layers=1):
|
||||
logical_to_physical = torch.tensor([[2, 0, 3, 1]], dtype=torch.int64).repeat(
|
||||
num_layers, 1
|
||||
)
|
||||
physical_to_logical = logical_to_physical.argsort(dim=-1)
|
||||
all_physical = torch.full((num_layers, 4, 4), -1, dtype=torch.int64)
|
||||
all_physical[:, :, 0] = logical_to_physical
|
||||
return ExpertLocationMetadata(
|
||||
physical_to_logical_map=physical_to_logical,
|
||||
physical_to_logical_map_cpu=physical_to_logical.clone(),
|
||||
logical_to_all_physical_map=all_physical,
|
||||
logical_to_all_physical_map_cpu=all_physical.clone(),
|
||||
logical_to_all_physical_map_num_valid=torch.ones(
|
||||
(num_layers, 4), dtype=torch.int64
|
||||
),
|
||||
ep_size=1,
|
||||
logical_to_rank_dispatch_physical_map=logical_to_physical,
|
||||
)
|
||||
|
||||
|
||||
class TestTopKRoutingInputs(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
config = get_context().override_server_args(
|
||||
ep_dispatch_algorithm="static", enable_eplb=True
|
||||
)
|
||||
config.install()
|
||||
self.addCleanup(config.restore)
|
||||
initialize_moe_config()
|
||||
topology = get_parallel().override(
|
||||
attn_dp_size=1, attn_cp_size=1, attn_tp_size=1, attn_tp_rank=0
|
||||
)
|
||||
topology.__enter__()
|
||||
self.addCleanup(topology.__exit__, None, None, None)
|
||||
for name in ("_is_cuda", "_is_hip", "_use_aiter"):
|
||||
scope = patch.object(topk_module, name, False)
|
||||
scope.start()
|
||||
self.addCleanup(scope.stop)
|
||||
self.metadata = _placement()
|
||||
get_resources().expert_location_metadata = self.metadata
|
||||
self.addCleanup(setattr, get_resources(), "expert_location_metadata", None)
|
||||
self.hidden = torch.ones((4, 2))
|
||||
self.logits = torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(4, 1)
|
||||
|
||||
def _router(self, **kwargs):
|
||||
shared_slots = kwargs.pop("num_fused_shared_experts", 0)
|
||||
layer_id = kwargs.pop("layer_id", 0)
|
||||
with get_model().override(
|
||||
shared_experts=SharedExpertsConfig(num_fused_shared_experts=shared_slots)
|
||||
):
|
||||
return TopK(
|
||||
top_k=2 - shared_slots,
|
||||
layer_id=layer_id,
|
||||
custom_routing_function=fused_topk_torch_native,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def test_target_and_draft_remap_only_the_target_and_keep_mask(self):
|
||||
"""Draft routes must not use the target placement or lose every live row."""
|
||||
target = self._router()
|
||||
with draft_model_build_scope():
|
||||
draft = self._router()
|
||||
batch = _batch(2)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=batch
|
||||
):
|
||||
target_ids = target.forward_native(self.hidden, self.logits).topk_ids
|
||||
draft_ids = draft.forward_native(self.hidden, self.logits).topk_ids
|
||||
self.assertEqual(target_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]])
|
||||
self.assertEqual(draft_ids.tolist(), [[0, 1], [0, 1], [-1, -1], [-1, -1]])
|
||||
self.assertFalse(draft.topk_config.allow_routed_experts_capture)
|
||||
self.assertTrue(target.topk_config.allow_routed_experts_capture)
|
||||
|
||||
def test_late_preparation_and_in_place_rebalance(self):
|
||||
"""Forward preparation and rebalance must be observed after scope entry."""
|
||||
router = self._router()
|
||||
batch = _batch(1)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=batch
|
||||
):
|
||||
before = router.forward_native(self.hidden, self.logits).topk_ids
|
||||
batch.num_token_non_padded = torch.tensor(3)
|
||||
self.metadata.logical_to_rank_dispatch_physical_map.copy_(
|
||||
torch.tensor([[1, 3, 0, 2]])
|
||||
)
|
||||
after = router.forward_native(self.hidden, self.logits).topk_ids
|
||||
self.assertEqual(before[0].tolist(), [2, 0])
|
||||
self.assertEqual(before[1].tolist(), [-1, -1])
|
||||
self.assertEqual(after.tolist(), [[1, 3], [1, 3], [1, 3], [-1, -1]])
|
||||
|
||||
def test_nested_scope_restores_and_retains_replay_buffer_identity(self):
|
||||
router = self._router()
|
||||
parent, child = _batch(3), _batch(1)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=parent
|
||||
):
|
||||
original = get_forward_context()
|
||||
bound, _ = router.routing_inputs()
|
||||
self.assertIs(bound, parent.num_token_non_padded)
|
||||
with self.assertRaisesRegex(RuntimeError, "child failed"):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=child
|
||||
):
|
||||
self.assertIs(
|
||||
router.routing_inputs()[0], child.num_token_non_padded
|
||||
)
|
||||
raise RuntimeError("child failed")
|
||||
self.assertIs(get_forward_context(), original)
|
||||
parent.num_token_non_padded.fill_(2)
|
||||
self.assertIs(router.routing_inputs()[0], bound)
|
||||
self.assertEqual(
|
||||
router.forward_native(self.hidden, self.logits).topk_ids[-2:].tolist(),
|
||||
[[-1, -1], [-1, -1]],
|
||||
)
|
||||
self.assertFalse(has_forward_context())
|
||||
|
||||
def test_tbo_children_bind_bounds_without_tbo_attention_backend(self):
|
||||
router = self._router()
|
||||
seen = []
|
||||
|
||||
def read_bound(state, forward_batch):
|
||||
seen.append(int(router.routing_inputs()[0]))
|
||||
return {"forward_batch": forward_batch}
|
||||
|
||||
stages = [[ExecutionOperation("route", read_bound)]] * 2
|
||||
parent, child_a, child_b = _batch(4), _batch(1), _batch(3)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=parent
|
||||
):
|
||||
a = _StageExecutor("a", stages, {"forward_batch": child_a})
|
||||
b = _StageExecutor("b", stages, {"forward_batch": child_b})
|
||||
a.next()
|
||||
b.next()
|
||||
a.next()
|
||||
b.next()
|
||||
self.assertEqual(int(router.routing_inputs()[0]), 4)
|
||||
self.assertEqual(seen, [1, 3, 1, 3])
|
||||
|
||||
def test_hash_router_uses_same_placement_and_bound(self):
|
||||
router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0)
|
||||
with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
|
||||
):
|
||||
output = router(self.hidden, self.logits, torch.arange(4))
|
||||
self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [-1, -1], [-1, -1]])
|
||||
|
||||
def test_vision_router_maps_routed_columns_and_masks_shared_rows_last(self):
|
||||
router = self._router(
|
||||
num_fused_shared_experts=1,
|
||||
routed_scaling_factor=2.0,
|
||||
fused_shared_experts_scaling_factor=0.25,
|
||||
)
|
||||
moe = SimpleNamespace(
|
||||
topk=router,
|
||||
layer_id=0,
|
||||
config=SimpleNamespace(image_token_id=99),
|
||||
gate=SimpleNamespace(
|
||||
e_score_correction_bias=torch.zeros(4),
|
||||
e_score_correction_bias_vl=torch.zeros(4),
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False
|
||||
):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
|
||||
):
|
||||
output = vision_topk(moe, self.logits, torch.tensor([1, 99, 1, 99]))
|
||||
self.assertEqual(output.topk_ids.tolist(), [[2, 4], [2, 4], [-1, -1], [-1, -1]])
|
||||
torch.testing.assert_close(output.topk_weights[:2, -1], torch.full((2,), 0.125))
|
||||
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
|
||||
|
||||
def test_vision_packed_output_follows_built_runner(self):
|
||||
"""A requested MXFP4 fallback must not emit an unused packed-ID buffer."""
|
||||
with get_context().override_server_args(moe_runner_backend="flashinfer_mxfp4"):
|
||||
initialize_moe_config()
|
||||
for backend in (
|
||||
None,
|
||||
MoeRunnerBackend.TRITON,
|
||||
MoeRunnerBackend.FLASHINFER_MXFP4,
|
||||
):
|
||||
with self.subTest(backend=backend):
|
||||
router = self._router(is_draft=True)
|
||||
router.bind_runner_backend(backend)
|
||||
moe = SimpleNamespace(
|
||||
topk=router,
|
||||
layer_id=0,
|
||||
config=SimpleNamespace(image_token_id=99),
|
||||
gate=SimpleNamespace(
|
||||
e_score_correction_bias=torch.zeros(4),
|
||||
e_score_correction_bias_vl=torch.zeros(4),
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"sglang.kernels.ops.moe.moe_fused_gate.moe_fused_gate",
|
||||
return_value=(
|
||||
torch.full((4, 2), 0.5),
|
||||
torch.tensor([[0, 1]], dtype=torch.int32).repeat(4, 1),
|
||||
),
|
||||
) as gate,
|
||||
):
|
||||
output = vision_topk(moe, self.logits, input_ids=None)
|
||||
packed = gate.call_args.kwargs["packed_out"]
|
||||
expected_packed = backend == MoeRunnerBackend.FLASHINFER_MXFP4
|
||||
self.assertEqual(packed is not None, expected_packed)
|
||||
self.assertEqual(
|
||||
isinstance(output, StandardTopKOutputPacked), expected_packed
|
||||
)
|
||||
|
||||
def test_vision_fake_algorithm_neutralizes_both_biases(self):
|
||||
with get_exec().moe.override(ep_dispatch_algorithm="fake"):
|
||||
router = self._router()
|
||||
moe = SimpleNamespace(
|
||||
topk=router,
|
||||
layer_id=0,
|
||||
config=SimpleNamespace(image_token_id=99),
|
||||
gate=SimpleNamespace(
|
||||
e_score_correction_bias=torch.tensor([0.0, 0.0, 1000.0, 0.0]),
|
||||
e_score_correction_bias_vl=torch.tensor([0.0, 0.0, 0.0, 1000.0]),
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"sglang.srt.multimodal.dsv41.vl_routing.is_cuda", return_value=False
|
||||
):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(4)
|
||||
):
|
||||
logits = self.logits.clone()
|
||||
output = vision_topk(moe, logits, torch.tensor([1, 99, 1, 99]))
|
||||
logical = logits.topk(2, dim=-1).indices
|
||||
expected = self.metadata.logical_to_all_physical_map[0, logical, 0]
|
||||
self.assertTrue(torch.equal(output.topk_ids.long(), expected))
|
||||
|
||||
def test_draft_without_input_ids_retains_prepared_live_width(self):
|
||||
"""Deferred draft IDs must not turn a live draft's padding bound into zero."""
|
||||
batch = _batch(0)
|
||||
batch.input_ids = None
|
||||
batch.spec_info = EagleDraftInput(num_tokens_per_req=2)
|
||||
with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4):
|
||||
batch._init_num_token_non_padded(device="cpu", is_draft_worker=True)
|
||||
self.assertEqual(batch.global_num_token_non_padded_cpu, 8)
|
||||
self.assertEqual(int(batch.global_num_token_non_padded), 8)
|
||||
batch.forward_mode = ForwardMode.IDLE
|
||||
with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4):
|
||||
batch._init_num_token_non_padded(device="cpu", is_draft_worker=True)
|
||||
self.assertEqual(int(batch.global_num_token_non_padded), 0)
|
||||
|
||||
def test_npu_routes_keep_valid_ids_and_zero_only_padded_contributions(self):
|
||||
"""NPU padding must stay inert without inventing invalid or duplicate routes."""
|
||||
router = self._router(num_fused_shared_experts=1)
|
||||
router.topk_config.scoring_func = "sqrtsoftplus"
|
||||
logical = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32)
|
||||
weights = torch.tensor([[0.75, 0.25]]).repeat(4, 1)
|
||||
|
||||
class Capture:
|
||||
def capture(self, *, layer_id, topk_indices):
|
||||
self.ids = topk_indices.clone()
|
||||
|
||||
capture = Capture()
|
||||
get_resources().experts_capturer = capture
|
||||
self.addCleanup(setattr, get_resources(), "experts_capturer", None)
|
||||
_, info = router.routing_inputs()
|
||||
with patch.object(
|
||||
torch.ops.custom,
|
||||
"npu_moe_gating_top_k",
|
||||
create=True,
|
||||
return_value=(weights, logical, None),
|
||||
):
|
||||
output = fused_topk_npu(
|
||||
hidden_states=self.hidden,
|
||||
router_logits=self.logits,
|
||||
topk_config=router.topk_config,
|
||||
num_token_non_padded=torch.tensor(2),
|
||||
expert_location_dispatch_info=info,
|
||||
layer_id=0,
|
||||
)
|
||||
self.assertEqual(output.topk_ids.tolist(), [[2, 4], [0, 4], [3, 4], [1, 4]])
|
||||
torch.testing.assert_close(
|
||||
output.topk_weights[:2], torch.tensor([[0.75, 0.25]]).repeat(2, 1)
|
||||
)
|
||||
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
|
||||
self.assertEqual(capture.ids.tolist(), [[0], [1], [2], [3]])
|
||||
|
||||
def test_in_kernel_formats_do_not_silently_ignore_expert_placement(self):
|
||||
router = self._router(output_format=TopKOutputFormat.TRITON_KERNEL)
|
||||
with self.assertRaisesRegex(
|
||||
NotImplementedError, "placement remapping requires"
|
||||
):
|
||||
router.forward_cuda(self.hidden, self.logits)
|
||||
|
||||
router = self._router(output_format=TopKOutputFormat.BYPASSED)
|
||||
output = router.forward_cuda(self.hidden, self.logits)
|
||||
with self.assertRaisesRegex(
|
||||
NotImplementedError, "placement remapping requires"
|
||||
):
|
||||
_ = output.router_logits
|
||||
|
||||
def test_bypassed_materialization_preserves_placement_and_padding(self):
|
||||
"""A dispatcher can remap lazy routes before sending physical expert IDs."""
|
||||
router = self._router(output_format=TopKOutputFormat.BYPASSED)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
|
||||
):
|
||||
output = router.forward_cuda(self.hidden, self.logits)
|
||||
materialized = output.to_standard(layer_id=0)
|
||||
self.assertEqual(
|
||||
materialized.topk_ids.tolist(), [[2, 0], [2, 0], [-1, -1], [-1, -1]]
|
||||
)
|
||||
self.assertIs(materialized.router_logits, self.logits)
|
||||
layer = SimpleNamespace(
|
||||
_use_ascend_fuseep=False,
|
||||
forward_impl=lambda hidden, routes, **kwargs: routes.to_standard(
|
||||
layer_id=0
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"sglang.srt.layers.moe.fused_moe_triton.layer.is_in_tc_piecewise_cuda_graph",
|
||||
return_value=True,
|
||||
):
|
||||
piecewise = FusedMoE.forward(layer, self.hidden, output)
|
||||
torch.testing.assert_close(piecewise.topk_ids, materialized.topk_ids)
|
||||
|
||||
def test_in_range_draft_loads_experts_in_its_routing_order(self):
|
||||
"""Draft expert weights stay in logical order despite a permuted target map."""
|
||||
outputs = []
|
||||
for is_draft in (False, True):
|
||||
method = UnquantizedFusedMoEMethod()
|
||||
scope = draft_model_build_scope() if is_draft else get_parallel().override()
|
||||
with (
|
||||
scope,
|
||||
patch.object(method, "create_weights"),
|
||||
get_parallel().override(
|
||||
tp_size=1,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
),
|
||||
):
|
||||
layer = FusedMoE(
|
||||
num_experts=4,
|
||||
hidden_size=2,
|
||||
intermediate_size=2,
|
||||
layer_id=0,
|
||||
top_k=2,
|
||||
quant_method=method,
|
||||
)
|
||||
weights = torch.nn.Parameter(torch.zeros((4, 2, 2)), requires_grad=False)
|
||||
for expert_id in range(4):
|
||||
layer.weight_loader(
|
||||
param=weights,
|
||||
loaded_weight=torch.eye(2) * (expert_id + 1),
|
||||
weight_name="experts.w2_weight",
|
||||
shard_id="w2",
|
||||
expert_id=expert_id,
|
||||
)
|
||||
self.assertEqual(
|
||||
weights[:, 0, 0].tolist(), [1, 2, 3, 4] if is_draft else [2, 4, 1, 3]
|
||||
)
|
||||
router = self._router(is_draft=is_draft)
|
||||
topk = router.forward_native(self.hidden, self.logits)
|
||||
expert_outputs = torch.einsum(
|
||||
"tkij,tj->tki", weights[topk.topk_ids], self.hidden
|
||||
)
|
||||
outputs.append((expert_outputs * topk.topk_weights[..., None]).sum(dim=1))
|
||||
torch.testing.assert_close(outputs[0], outputs[1])
|
||||
expected_scale = torch.softmax(self.logits[:, :2], dim=-1) @ torch.tensor(
|
||||
[1.0, 2.0]
|
||||
)
|
||||
torch.testing.assert_close(outputs[1], self.hidden * expected_scale[:, None])
|
||||
|
||||
def test_identity_placement_keeps_in_kernel_routing_available(self):
|
||||
identity = torch.arange(4).view(1, 4)
|
||||
self.metadata.logical_to_rank_dispatch_physical_map.copy_(identity)
|
||||
self.metadata.logical_to_all_physical_map[:, :, 0].copy_(identity)
|
||||
self.metadata.physical_to_logical_map.copy_(identity)
|
||||
with get_exec().moe.override(
|
||||
enable_eplb=False,
|
||||
init_expert_location="trivial",
|
||||
ep_num_redundant_experts=0,
|
||||
ep_dispatch_algorithm="dynamic",
|
||||
):
|
||||
for output_format in (
|
||||
TopKOutputFormat.BYPASSED,
|
||||
TopKOutputFormat.TRITON_KERNEL,
|
||||
):
|
||||
with self.subTest(output_format=output_format):
|
||||
router = self._router(output_format=output_format)
|
||||
with patch.object(
|
||||
topk_module, "routing", return_value=(None, None, None, None, 2)
|
||||
):
|
||||
output = router.forward_cuda(self.hidden, self.logits)
|
||||
self.assertEqual(output.format, output_format)
|
||||
if output_format == TopKOutputFormat.BYPASSED:
|
||||
self.assertIs(output.router_logits, self.logits)
|
||||
for algorithm in ("fake", "lp"):
|
||||
with (
|
||||
self.subTest(algorithm=algorithm),
|
||||
get_exec().moe.override(ep_dispatch_algorithm=algorithm),
|
||||
):
|
||||
router = self._router(output_format=TopKOutputFormat.BYPASSED)
|
||||
output = router.forward_cuda(self.hidden, self.logits)
|
||||
with self.assertRaisesRegex(
|
||||
NotImplementedError, "placement remapping requires"
|
||||
):
|
||||
_ = output.router_logits
|
||||
|
||||
def test_hash_npu_keeps_valid_ids_and_zero_padded_weights(self):
|
||||
router = HashTopK(2, 4, vocab_size=4, scoring_func="softmax", layer_id=0)
|
||||
with (
|
||||
patch.object(hash_topk_module, "_is_npu", True),
|
||||
envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False),
|
||||
):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
|
||||
):
|
||||
output = router(self.hidden, self.logits, torch.arange(4))
|
||||
self.assertEqual(output.topk_ids.tolist(), [[2, 0], [0, 3], [3, 1], [1, 2]])
|
||||
torch.testing.assert_close(output.topk_weights[2:], torch.zeros((2, 2)))
|
||||
|
||||
def test_hash_global_shared_column_stays_outside_expert_map(self):
|
||||
with get_model().override(
|
||||
shared_experts=SharedExpertsConfig(num_fused_shared_experts=1)
|
||||
):
|
||||
router = HashTopK(
|
||||
2,
|
||||
4,
|
||||
vocab_size=4,
|
||||
scoring_func="sqrtsoftplus",
|
||||
routed_scaling_factor=2.0,
|
||||
layer_id=0,
|
||||
)
|
||||
with envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.override(False):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=_batch(2)
|
||||
):
|
||||
output = router(self.hidden, self.logits, torch.arange(4))
|
||||
self.assertEqual(
|
||||
output.topk_ids.tolist(), [[2, 0, 4], [0, 3, 4], [-1, -1, -1], [-1, -1, -1]]
|
||||
)
|
||||
|
||||
def test_deepseek_opaque_bypass_requires_identity_placement_and_native_policy(self):
|
||||
from sglang.srt.models import deepseek_v2
|
||||
|
||||
moe = SimpleNamespace(
|
||||
alt_stream=object(),
|
||||
num_fused_shared_experts=0,
|
||||
shared_experts=object(),
|
||||
experts=SimpleNamespace(use_flashinfer_trtllm_moe=True),
|
||||
_enable_a2a_moe=False,
|
||||
_fuse_shared_experts_inside_sbo=False,
|
||||
is_hash=False,
|
||||
topk=self._router(output_format=TopKOutputFormat.BYPASSED),
|
||||
)
|
||||
with (
|
||||
patch.object(deepseek_v2, "_enable_pcg_dsv2_dual_stream", True),
|
||||
patch.object(
|
||||
deepseek_v2, "is_in_tc_piecewise_cuda_graph", return_value=True
|
||||
),
|
||||
patch.object(
|
||||
deepseek_v2,
|
||||
"get_moe_runner_backend",
|
||||
return_value=MoeRunnerBackend.FLASHINFER_TRTLLM,
|
||||
),
|
||||
get_exec().moe.override(
|
||||
enable_eplb=False,
|
||||
init_expert_location="trivial",
|
||||
ep_num_redundant_experts=0,
|
||||
ep_dispatch_algorithm=None,
|
||||
),
|
||||
):
|
||||
self.assertTrue(
|
||||
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
|
||||
)
|
||||
with get_exec().moe.override(init_expert_location="reverse"):
|
||||
self.assertFalse(
|
||||
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
|
||||
)
|
||||
moe.topk.topk_config.output_format = TopKOutputFormat.STANDARD
|
||||
self.assertFalse(
|
||||
deepseek_v2.DeepseekV2MoE._can_dual_stream_graph(moe, self.hidden)
|
||||
)
|
||||
|
||||
def test_lp_keeps_global_shared_columns_outside_routed_domain(self):
|
||||
router = self._router(num_fused_shared_experts=1)
|
||||
ids = torch.tensor([[0, 4], [1, 4], [2, 4], [3, 4]], dtype=torch.int32)
|
||||
weights = torch.tensor([[0.8, 0.2]]).repeat(4, 1)
|
||||
solved = []
|
||||
|
||||
def solve(routed_ids):
|
||||
self.assertEqual(routed_ids.tolist(), [[0], [1], [2], [3]])
|
||||
solved.append(routed_ids.clone())
|
||||
return torch.ones(4, 1)
|
||||
|
||||
def dispatch(routed_ids, info, probability=None):
|
||||
self.assertTrue(bool(((routed_ids >= 0) & (routed_ids < 4)).all()))
|
||||
return info.partial_logical_to_rank_dispatch_physical_map[routed_ids]
|
||||
|
||||
get_resources().lplb_solvers[0] = SimpleNamespace(solve=solve)
|
||||
self.addCleanup(get_resources().lplb_solvers.pop, 0)
|
||||
with (
|
||||
get_exec().moe.override(ep_dispatch_algorithm="lp"),
|
||||
patch.object(topk_module, "_is_cuda", True),
|
||||
patch.object(topk_module, "topk_ids_logical_to_physical", dispatch),
|
||||
):
|
||||
_, info = router.routing_inputs()
|
||||
output_ids, output_weights, _ = topk_module._post_process_topk_ids(
|
||||
ids,
|
||||
weights,
|
||||
topk_config=router.topk_config,
|
||||
router_logits=self.logits,
|
||||
layer_id=0,
|
||||
num_token_non_padded=torch.tensor(2),
|
||||
expert_location_dispatch_info=info,
|
||||
)
|
||||
self.assertEqual(len(solved), 1)
|
||||
self.assertEqual(output_ids.tolist(), [[2, 4], [0, 4], [-1, -1], [-1, -1]])
|
||||
torch.testing.assert_close(output_weights[:2], weights[:2])
|
||||
|
||||
def test_lazy_lp_materialization_uses_its_owner_or_explicit_override(self):
|
||||
"""Lazy dispatcher materialization must retain the layer's solver and capture slot."""
|
||||
get_resources().expert_location_metadata = _placement(num_layers=8)
|
||||
hidden = self.hidden.to(torch.bfloat16)
|
||||
group = SimpleNamespace(all_gatherv=lambda tensors, sizes: tensors)
|
||||
dispatcher = object.__new__(FlashinferDispatcher)
|
||||
dispatcher.ep_size = 1
|
||||
dispatcher.ep_rank = 0
|
||||
with get_parallel().override(moe_ep_size=1, moe_ep_rank=0):
|
||||
dispatcher.prefill_dispatcher = StandardDispatcher(
|
||||
SimpleNamespace(
|
||||
runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM,
|
||||
num_experts=4,
|
||||
num_local_experts=4,
|
||||
num_fused_shared_experts=0,
|
||||
)
|
||||
)
|
||||
for owner, override, expected_owner in (
|
||||
(7, None, 7),
|
||||
(0, None, 0),
|
||||
(7, 0, 0),
|
||||
(0, 7, 7),
|
||||
):
|
||||
with self.subTest(owner=owner, override=override):
|
||||
solved = []
|
||||
captured = {}
|
||||
|
||||
def solver(layer_id):
|
||||
def solve(ids):
|
||||
self.assertEqual(ids.tolist(), [[0, 1]] * 4)
|
||||
solved.append(layer_id)
|
||||
return torch.ones((4, 1))
|
||||
|
||||
return SimpleNamespace(solve=solve)
|
||||
|
||||
class Capture:
|
||||
def capture(self, *, layer_id, topk_indices):
|
||||
captured[layer_id] = topk_indices.clone()
|
||||
|
||||
def dispatch_probability(ids, info, probability):
|
||||
return info.partial_logical_to_rank_dispatch_physical_map[ids]
|
||||
|
||||
with (
|
||||
get_exec().moe.override(ep_dispatch_algorithm="lp"),
|
||||
get_parallel().override(
|
||||
tp_group=group, moe_ep_size=1, moe_ep_rank=0
|
||||
),
|
||||
patch.dict(
|
||||
get_resources().lplb_solvers,
|
||||
{0: solver(0), 7: solver(7)},
|
||||
clear=True,
|
||||
),
|
||||
patch.object(get_resources(), "experts_capturer", Capture()),
|
||||
patch.object(topk_module, "_is_cuda", True),
|
||||
patch.object(
|
||||
expert_location_dispatch,
|
||||
"_topk_ids_logical_to_physical_probability",
|
||||
dispatch_probability,
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.moe.token_dispatcher.flashinfer.get_dp_global_num_tokens",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
router = self._router(
|
||||
layer_id=owner, output_format=TopKOutputFormat.BYPASSED
|
||||
)
|
||||
output = router.forward_cuda(
|
||||
hidden, self.logits, num_token_non_padded=torch.tensor(2)
|
||||
)
|
||||
self.assertEqual(len(output), 5)
|
||||
if override is not None:
|
||||
output = output.to_standard(layer_id=override)
|
||||
dispatched = dispatcher._dispatch_prefill_allgather(hidden, output)
|
||||
self.assertEqual(solved, [expected_owner])
|
||||
self.assertEqual(set(captured), {expected_owner})
|
||||
self.assertEqual(captured[expected_owner].tolist(), [[0, 1]] * 4)
|
||||
self.assertEqual(
|
||||
dispatched.topk_output.topk_ids.tolist(),
|
||||
[[2, 0], [2, 0], [-1, -1], [-1, -1]],
|
||||
)
|
||||
self.assertIs(dispatched.hidden_states, hidden)
|
||||
|
||||
def test_empty_target_participates_in_lp_without_caller_layer_id(self):
|
||||
class Solver:
|
||||
def __init__(self):
|
||||
self.participations = []
|
||||
|
||||
def solve(self, ids):
|
||||
self.participations.append(tuple(ids.shape))
|
||||
|
||||
solver = Solver()
|
||||
get_resources().lplb_solvers[0] = solver
|
||||
self.addCleanup(get_resources().lplb_solvers.pop, 0)
|
||||
target = self._router()
|
||||
with draft_model_build_scope():
|
||||
draft = self._router()
|
||||
with get_parallel().override(tp_group=SimpleNamespace(world_size=1)):
|
||||
output = target.empty_topk_output(torch.device("cpu"))
|
||||
draft_output = draft.empty_topk_output(torch.device("cpu"))
|
||||
self.assertEqual(output.topk_ids.shape, (0, 2))
|
||||
self.assertEqual(draft_output.topk_ids.shape, (0, 2))
|
||||
self.assertEqual(solver.participations, [(0, 2)])
|
||||
|
||||
def test_compile_reads_current_buffer(self):
|
||||
router = self._router(is_draft=True)
|
||||
batch = _batch(3)
|
||||
|
||||
def mask_rows(x):
|
||||
bound, _ = router.routing_inputs()
|
||||
return x * (torch.arange(x.shape[0]) < bound)
|
||||
|
||||
compiled = torch.compile(mask_rows, backend="eager", fullgraph=True)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=batch
|
||||
):
|
||||
self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 1.0, 1.0, 0.0])
|
||||
batch.num_token_non_padded.fill_(1)
|
||||
self.assertEqual(compiled(torch.ones(4)).tolist(), [1.0, 0.0, 0.0, 0.0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,342 @@
|
||||
"""GGUF expert kernels drop inactive routes during padded graph replay."""
|
||||
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from gguf import GGMLQuantizationType, dequantize, quantize
|
||||
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
|
||||
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend
|
||||
from sglang.srt.layers.quantization import gguf as gguf_runtime
|
||||
from sglang.srt.layers.quantization.gguf import (
|
||||
GGUFConfig,
|
||||
GGUFMoEMethod,
|
||||
fused_moe_gguf,
|
||||
ggml_moe_get_block_size,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import ForwardContext, forward_context
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestGgufInactiveRoutes(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.enterContext(
|
||||
get_context().override_server_args(model_path="dummy", device="cuda")
|
||||
)
|
||||
self.enterContext(
|
||||
get_flags().moe.override(
|
||||
runner_backend=MoeRunnerBackend.AUTO, a2a_backend=MoeA2ABackend.NONE
|
||||
)
|
||||
)
|
||||
self.enterContext(
|
||||
get_parallel().override(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
attn_tp_size=1,
|
||||
attn_tp_rank=0,
|
||||
attn_dp_size=1,
|
||||
attn_cp_size=1,
|
||||
tp_group=None,
|
||||
moe_tp_size=1,
|
||||
moe_tp_rank=0,
|
||||
moe_ep_size=1,
|
||||
moe_ep_rank=0,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _weights(quant_type):
|
||||
rng = np.random.default_rng(491)
|
||||
shape13, shape2 = (4, 512, 256), (4, 256, 256)
|
||||
packed, dense = [], []
|
||||
for shape in (shape13, shape2):
|
||||
weight = rng.normal(0, 0.03, shape).astype(np.float32)
|
||||
qweight = quantize(weight, quant_type)
|
||||
packed.append(torch.from_numpy(qweight.copy()).cuda())
|
||||
dense.append(torch.from_numpy(dequantize(qweight, quant_type)).cuda())
|
||||
return packed, dense
|
||||
|
||||
@staticmethod
|
||||
def _method(packed, quant_type, *, global_experts=4):
|
||||
method = GGUFMoEMethod(GGUFConfig())
|
||||
layer = torch.nn.Module()
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=4,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=256,
|
||||
params_dtype=torch.float16,
|
||||
)
|
||||
for param, weight in zip((layer.w13_qweight, layer.w2_qweight), packed):
|
||||
param.materialize(weight.shape, device=weight.device, dtype=weight.dtype)
|
||||
param.data.copy_(weight)
|
||||
layer.w13_qweight_type.weight_type = int(quant_type)
|
||||
layer.w2_qweight_type.weight_type = int(quant_type)
|
||||
config = MoeRunnerConfig(
|
||||
num_experts=global_experts,
|
||||
num_local_experts=4,
|
||||
num_fused_shared_experts=0,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=256,
|
||||
top_k=2,
|
||||
)
|
||||
method.create_moe_runner(layer, config)
|
||||
config.runner_backend = method.get_moe_runner_backend(layer)
|
||||
return method, layer, StandardDispatcher(config)
|
||||
|
||||
@staticmethod
|
||||
def _q8_1_matmul(inputs, weight, packed_weight, quant_type):
|
||||
# GGUF GEMMs quantize each 32 input values to Q8_1, including an FP16
|
||||
# scale and input sum. CUDA roundf rounds halfway values away from zero.
|
||||
blocks = inputs.float().unflatten(-1, (-1, 32))
|
||||
scale = blocks.abs().amax(dim=-1, keepdim=True) / 127
|
||||
denominator = torch.where(scale == 0, 1.0, scale)
|
||||
codes = blocks.sign() * torch.floor(blocks.abs() / denominator + 0.5)
|
||||
quantized = codes * scale.half().float()
|
||||
result = torch.einsum(
|
||||
"...bi,...nbi->...n", quantized, weight.unflatten(-1, (-1, 32))
|
||||
)
|
||||
if quant_type == GGMLQuantizationType.Q4_0:
|
||||
# Q4_0 subtracts its zero point using the original FP16 input sum,
|
||||
# rather than summing the dequantized Q8_1 codes.
|
||||
weight_scale = (
|
||||
packed_weight.unflatten(-1, (-1, 18))[..., :2]
|
||||
.contiguous()
|
||||
.view(torch.float16)
|
||||
.squeeze(-1)
|
||||
.float()
|
||||
)
|
||||
sum_error = quantized.sum(dim=-1) - blocks.sum(dim=-1).half().float()
|
||||
result += 8 * torch.einsum("...b,...nb->...n", sum_error, weight_scale)
|
||||
return result.to(inputs.dtype)
|
||||
|
||||
@classmethod
|
||||
def _reference(cls, hidden, logits, dense, packed, quant_type):
|
||||
weights, ids = logits.float().softmax(dim=-1).topk(2, dim=-1)
|
||||
projected = cls._q8_1_matmul(
|
||||
hidden.unsqueeze(1), dense[0][ids], packed[0][ids], quant_type
|
||||
)
|
||||
gate, up = projected.float().chunk(2, dim=-1)
|
||||
activated = (torch.nn.functional.silu(gate) * up).to(hidden.dtype)
|
||||
down = cls._q8_1_matmul(activated, dense[1][ids], packed[1][ids], quant_type)
|
||||
weighted = (down.float() * weights.unsqueeze(-1)).to(hidden.dtype)
|
||||
return weighted.sum(dim=1).float()
|
||||
|
||||
def test_block_size_metadata_without_tensor_arguments(self):
|
||||
# Runtime metadata must work without a Tensor-dispatched wheel operator.
|
||||
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
|
||||
with self.subTest(quant_type=quant_type):
|
||||
self.assertIn(ggml_moe_get_block_size(int(quant_type)), (4, 8))
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_padded_graph_replay_matches_unpadded_experts(self):
|
||||
"""Changing the live count must zero dropped GGUF routes on every replay."""
|
||||
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
|
||||
packed, dense = self._weights(quant_type)
|
||||
for tokens, partial in ((64, 48), (96, 71)):
|
||||
with self.subTest(quant_type=quant_type, tokens=tokens):
|
||||
self._check_graph(packed, dense, quant_type, tokens, partial)
|
||||
|
||||
def _check_graph(self, packed, dense, quant_type, tokens, partial):
|
||||
method, layer, dispatcher = self._method(packed, quant_type)
|
||||
topk = TopK(
|
||||
top_k=2, layer_id=0, renormalize=False, allow_routed_experts_capture=False
|
||||
)
|
||||
topk.bind_runner_backend(method.get_moe_runner_backend(layer))
|
||||
generator = torch.Generator(device="cuda").manual_seed(tokens)
|
||||
base_hidden = torch.randn(
|
||||
tokens, 256, generator=generator, device="cuda"
|
||||
).half()
|
||||
base_logits = torch.randn(tokens, 4, generator=generator, device="cuda")
|
||||
logits = base_logits.clone()
|
||||
hidden = base_hidden.clone()
|
||||
live_count = torch.tensor(tokens, device="cuda", dtype=torch.int32)
|
||||
batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
batch_size=1,
|
||||
input_ids=torch.zeros(tokens, device="cuda", dtype=torch.int64),
|
||||
req_pool_indices=torch.zeros(1, device="cuda", dtype=torch.int64),
|
||||
seq_lens=torch.tensor([tokens], device="cuda", dtype=torch.int32),
|
||||
out_cache_loc=torch.arange(tokens, device="cuda"),
|
||||
seq_lens_sum=tokens,
|
||||
positions=torch.arange(tokens, device="cuda"),
|
||||
num_token_non_padded=live_count,
|
||||
global_num_token_non_padded=live_count,
|
||||
)
|
||||
|
||||
def run():
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=None), forward_batch=batch
|
||||
):
|
||||
routes = topk.forward_cuda(hidden, logits)
|
||||
dispatched = dispatcher.dispatch(hidden, routes)
|
||||
combined = method.apply(layer, dispatched)
|
||||
return dispatcher.combine(combined), routes.topk_ids
|
||||
|
||||
native_outputs = []
|
||||
native_mmq = gguf_runtime.ggml_moe_a8
|
||||
|
||||
native_align = gguf_runtime.moe_align_block_size
|
||||
|
||||
def poison_alignment(*args, **kwargs):
|
||||
sorted_ids, expert_ids, count = native_align(*args, **kwargs)
|
||||
block_starts = (
|
||||
torch.arange(expert_ids.numel(), device=expert_ids.device) * 4
|
||||
)
|
||||
expert_ids.masked_fill_(block_starts >= count, 4)
|
||||
return sorted_ids, expert_ids, count
|
||||
|
||||
def record_mmq_outputs(*args, **kwargs):
|
||||
if not torch.cuda.is_current_stream_capturing():
|
||||
# Fail before native dispatch if a slack block can index expert E.
|
||||
self.assertTrue(bool((args[3] < args[1].shape[0]).all()))
|
||||
result = native_mmq(*args, **kwargs)
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
native_outputs.append(result)
|
||||
return result
|
||||
|
||||
# Observe the real provider's returned buffers without replacing its
|
||||
# computation. Poisoning them makes a skipped MMQ store deterministic.
|
||||
observe_mmq = (
|
||||
patch.object(gguf_runtime, "ggml_moe_a8", record_mmq_outputs)
|
||||
if tokens > 64
|
||||
else nullcontext()
|
||||
)
|
||||
with (
|
||||
observe_mmq,
|
||||
patch.object(gguf_runtime, "moe_align_block_size", poison_alignment),
|
||||
):
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
run()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output, ids = run()
|
||||
self.assertEqual(len(native_outputs), 2 if tokens > 64 else 0)
|
||||
# Dropped slots must overwrite poisoned native intermediates rather
|
||||
# than inherit any previous live route or allocator contents.
|
||||
for live in (tokens, partial, 0, partial // 2, tokens):
|
||||
hidden.copy_(base_hidden)
|
||||
hidden[live:].fill_(float("nan"))
|
||||
logits.copy_(base_logits)
|
||||
logits[live:].fill_(float("nan"))
|
||||
live_count.fill_(live)
|
||||
output.fill_(float("nan"))
|
||||
for intermediate in native_outputs:
|
||||
intermediate.fill_(float("nan"))
|
||||
graph.replay()
|
||||
for intermediate in native_outputs:
|
||||
self.assertTrue(torch.isfinite(intermediate).all().item())
|
||||
inactive = intermediate.view(tokens, 2, -1)[live:]
|
||||
self.assertEqual(torch.count_nonzero(inactive).item(), 0)
|
||||
self.assertTrue(torch.isfinite(output).all().item())
|
||||
self.assertTrue((ids[live:] == -1).all().item())
|
||||
self.assertEqual(torch.count_nonzero(output[live:]).item(), 0)
|
||||
if live:
|
||||
expected = self._reference(
|
||||
base_hidden[:live], base_logits[:live], dense, packed, quant_type
|
||||
)
|
||||
self.assertGreater(expected.abs().max().item(), 1e-3)
|
||||
torch.testing.assert_close(
|
||||
output[:live].float(), expected, atol=2e-3, rtol=0.06
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_nonlocal_routes_preserve_local_contributions(self):
|
||||
"""A nonlocal slot must not erase the same token's valid expert result."""
|
||||
packed, dense = self._weights(GGMLQuantizationType.Q4_0)
|
||||
with get_parallel().override(
|
||||
tp_size=2,
|
||||
tp_rank=1,
|
||||
attn_tp_size=2,
|
||||
attn_tp_rank=1,
|
||||
moe_ep_size=2,
|
||||
moe_ep_rank=1,
|
||||
):
|
||||
method, layer, dispatcher = self._method(
|
||||
packed, GGMLQuantizationType.Q4_0, global_experts=8
|
||||
)
|
||||
hidden = torch.full((3, 256), 0.25, device="cuda", dtype=torch.float16)
|
||||
ids = torch.tensor([[4, 1], [-1, 5], [0, 2]], device="cuda", dtype=torch.int32)
|
||||
weights = torch.tensor(
|
||||
[[0.7, float("nan")], [float("nan"), 0.6], [float("nan"), float("nan")]],
|
||||
device="cuda",
|
||||
)
|
||||
hidden[2].fill_(float("nan"))
|
||||
routes = StandardTopKOutput(weights, ids, None)
|
||||
output = dispatcher.combine(
|
||||
method.apply(layer, dispatcher.dispatch(hidden, routes))
|
||||
)
|
||||
self.assertTrue(torch.isfinite(output).all().item())
|
||||
self.assertEqual(torch.count_nonzero(output[2]).item(), 0)
|
||||
for token, expert, scale in ((0, 0, 0.7), (1, 1, 0.6)):
|
||||
projected = self._q8_1_matmul(
|
||||
hidden[token],
|
||||
dense[0][expert],
|
||||
packed[0][expert],
|
||||
GGMLQuantizationType.Q4_0,
|
||||
)
|
||||
gate, up = projected.float().chunk(2)
|
||||
activated = (torch.nn.functional.silu(gate) * up).half()
|
||||
expected = (
|
||||
(
|
||||
self._q8_1_matmul(
|
||||
activated,
|
||||
dense[1][expert],
|
||||
packed[1][expert],
|
||||
GGMLQuantizationType.Q4_0,
|
||||
).float()
|
||||
* scale
|
||||
)
|
||||
.half()
|
||||
.float()
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[token].float(), expected, atol=2e-3, rtol=0.06
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_unquantized_fallback_drops_negative_ids(self):
|
||||
"""The slow path must not alias a dropped route to the last expert."""
|
||||
generator = torch.Generator(device="cuda").manual_seed(17)
|
||||
w13 = torch.randn(2, 32, 16, generator=generator, device="cuda").half() / 16
|
||||
w2 = torch.randn(2, 16, 16, generator=generator, device="cuda").half() / 16
|
||||
hidden = torch.ones(2, 16, device="cuda", dtype=torch.float16)
|
||||
hidden[1].fill_(float("nan"))
|
||||
ids = torch.tensor([[0, -1], [-1, -1]], device="cuda", dtype=torch.int32)
|
||||
weights = torch.tensor(
|
||||
[[1.0, float("nan")], [float("nan"), float("nan")]], device="cuda"
|
||||
)
|
||||
output = fused_moe_gguf(
|
||||
hidden,
|
||||
w13,
|
||||
w2,
|
||||
weights,
|
||||
ids,
|
||||
GGMLQuantizationType.F16,
|
||||
GGMLQuantizationType.F16,
|
||||
"silu",
|
||||
)
|
||||
projected = hidden[0] @ w13[0].t()
|
||||
gate, up = projected.chunk(2)
|
||||
expected = (torch.nn.functional.silu(gate) * up) @ w2[0].t()
|
||||
self.assertTrue(torch.isfinite(output).all().item())
|
||||
self.assertEqual(torch.count_nonzero(output[1]).item(), 0)
|
||||
torch.testing.assert_close(output[0], expected, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -196,7 +196,7 @@ class _MockDispatchOutput:
|
||||
self.hidden_states = hidden_states
|
||||
self.topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=TopKConfig(top_k=top_k, renormalize=True),
|
||||
)
|
||||
|
||||
|
||||
@@ -189,6 +189,7 @@ def _run_marlin_flow(
|
||||
fused_tails=0,
|
||||
marlin_is_ep=[],
|
||||
marlin_output_ids=[],
|
||||
marlin_output_initialized=[],
|
||||
align_num_experts=[],
|
||||
schedule=[],
|
||||
zeroed=[],
|
||||
@@ -248,6 +249,15 @@ def _run_marlin_flow(
|
||||
return _ORIGINAL_TENSOR_ZERO(tensor, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "zero_", tracked_zero)
|
||||
original_empty = torch.empty
|
||||
|
||||
def poisoned_empty(*args, **kwargs):
|
||||
result = original_empty(*args, **kwargs)
|
||||
if result.is_floating_point():
|
||||
result.fill_(float("nan"))
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(torch, "empty", poisoned_empty)
|
||||
|
||||
def merged_experts_fused_moe_lora_add(**kwargs):
|
||||
intermediate = kwargs.get("intermediate_buffer")
|
||||
@@ -298,11 +308,13 @@ def _run_marlin_flow(
|
||||
def fake_marlin_gemm(_x, output, *_args, **kwargs):
|
||||
calls.marlin_is_ep.append(kwargs["is_ep"])
|
||||
calls.marlin_output_ids.append(id(output))
|
||||
calls.marlin_output_initialized.append(
|
||||
bool(torch.isfinite(output).all() and torch.count_nonzero(output) == 0)
|
||||
)
|
||||
calls.schedule.append(
|
||||
("marlin", stream_state["current"].name, len(calls.marlin_is_ep))
|
||||
)
|
||||
# ``fill_``, never ``zero_``: the zero-fill contracts are asserted by
|
||||
# watching ``Tensor.zero_`` calls, so the fakes must not forge one.
|
||||
# Inspect initialization before the fake GEMM writes its result.
|
||||
output.fill_(0)
|
||||
return output
|
||||
|
||||
@@ -410,22 +422,14 @@ def _run_marlin_flow(
|
||||
|
||||
|
||||
def test_ep_uses_local_alignment_and_skips_nonlocal_marlin_blocks(monkeypatch):
|
||||
"""Guards expert-parallel index localization and both accumulator clears;
|
||||
reds when ``moe_align_block_size`` gets the global expert count, ``is_ep``
|
||||
is dropped on a Marlin GEMM, or an accumulator is left uninitialized."""
|
||||
"""EP alignment stays local and the down accumulator starts finite and zero."""
|
||||
|
||||
calls = _run_marlin_flow(monkeypatch, tokens=16, ep=True, slots=2, rank=128)
|
||||
|
||||
assert calls.align_num_experts == [1]
|
||||
assert calls.marlin_is_ep == [True, True]
|
||||
|
||||
down_gemm_output = calls.marlin_output_ids[1]
|
||||
assert [item[2] for item in calls.zeroed] == [down_gemm_output]
|
||||
zero_index = calls.schedule.index(("zero", "main", down_gemm_output))
|
||||
down_gemm_index = [
|
||||
index for index, item in enumerate(calls.schedule) if item[0] == "marlin"
|
||||
][1]
|
||||
assert zero_index < down_gemm_index
|
||||
assert calls.marlin_output_initialized[1]
|
||||
|
||||
down_stage = [
|
||||
call for call in calls.merged if call["stage"] == "all" and call["shared_b"]
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
"""Executor-owned routing bounds survive retained callbacks and padded CPU replay."""
|
||||
|
||||
import importlib
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.topk import TopK, fused_topk_torch_native
|
||||
from sglang.srt.layers.moe.utils import initialize_moe_config
|
||||
from sglang.srt.model_executor.cpu_graph_runner import CPUGraphRunner
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
PPProxyTensors,
|
||||
enable_num_token_non_padded,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
|
||||
DecodeCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend import (
|
||||
BreakableCudaGraphBackend,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
breakable_graph = importlib.import_module(
|
||||
"sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph"
|
||||
)
|
||||
breakable_backend = importlib.import_module(
|
||||
"sglang.srt.model_executor.runner_backend.breakable_cuda_graph_backend"
|
||||
)
|
||||
|
||||
|
||||
def _batch(size, live, *, prepared=True):
|
||||
batch = ForwardBatch(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
batch_size=size,
|
||||
input_ids=torch.arange(size),
|
||||
req_pool_indices=torch.arange(size),
|
||||
seq_lens=torch.ones(size, dtype=torch.int32),
|
||||
out_cache_loc=torch.arange(size),
|
||||
seq_lens_sum=size,
|
||||
positions=torch.arange(size),
|
||||
capture_hidden_mode=CaptureHiddenMode.NULL,
|
||||
)
|
||||
batch.global_num_token_non_padded = torch.tensor(live, dtype=torch.int32)
|
||||
batch.global_num_token_non_padded_cpu = live
|
||||
batch.num_token_non_padded = batch.global_num_token_non_padded if prepared else None
|
||||
batch.global_dp_buffer_len = size
|
||||
batch.dp_padding_mode = DpPaddingMode.SUM_LEN
|
||||
return batch
|
||||
|
||||
|
||||
def _attention_backend():
|
||||
return SimpleNamespace(
|
||||
init_forward_metadata=lambda batch: None,
|
||||
init_forward_metadata_out_graph=lambda batch, **kwargs: None,
|
||||
init_forward_metadata_in_graph=lambda batch: None,
|
||||
shared_read_ends=lambda mode: SharedReadEnds.UNKNOWN,
|
||||
)
|
||||
|
||||
|
||||
class _RoutingModel:
|
||||
shared_experts_config = SharedExpertsConfig()
|
||||
|
||||
def __init__(self, *, proxy_output):
|
||||
self.topk = TopK(
|
||||
2,
|
||||
layer_id=0,
|
||||
is_draft=True,
|
||||
custom_routing_function=fused_topk_torch_native,
|
||||
)
|
||||
self.proxy_output = proxy_output
|
||||
self.seen = []
|
||||
self.fail = False
|
||||
|
||||
def forward(self, input_ids, positions, batch):
|
||||
rows = len(input_ids)
|
||||
bound, _ = self.topk.routing_inputs()
|
||||
ids = self.topk.forward_native(
|
||||
torch.ones((rows, 2)),
|
||||
torch.tensor([[9.0, 8.0, 1.0, 0.0]]).repeat(rows, 1),
|
||||
).topk_ids
|
||||
self.seen.append(
|
||||
(batch, bound, get_forward_context().attn_backend, ids.clone())
|
||||
)
|
||||
if self.fail:
|
||||
raise RuntimeError("replayed model failed")
|
||||
if self.proxy_output:
|
||||
return PPProxyTensors({"routing_ids": ids})
|
||||
return LogitsProcessorOutput(next_token_logits=ids.float())
|
||||
|
||||
|
||||
class _CpuStream:
|
||||
def wait_stream(self, other):
|
||||
pass
|
||||
|
||||
|
||||
class _NoDeviceGraph:
|
||||
"""Only accelerator graph recording/replay is replaced; eager breaks remain real."""
|
||||
|
||||
def capture_begin(self, **kwargs):
|
||||
pass
|
||||
|
||||
def capture_end(self):
|
||||
pass
|
||||
|
||||
def replay(self):
|
||||
pass
|
||||
|
||||
|
||||
class TestGraphRoutingContext(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
config = get_context().override_server_args(
|
||||
enable_eplb=False, ep_dispatch_algorithm=None, debug_cuda_graph=True
|
||||
)
|
||||
config.install()
|
||||
self.addCleanup(config.restore)
|
||||
initialize_moe_config()
|
||||
topology = get_parallel().override(
|
||||
tp_size=4,
|
||||
moe_ep_size=4,
|
||||
attn_dp_size=1,
|
||||
attn_cp_size=1,
|
||||
attn_tp_size=4,
|
||||
attn_tp_rank=0,
|
||||
tp_group=SimpleNamespace(barrier=lambda: None, world_size=4),
|
||||
)
|
||||
topology.__enter__()
|
||||
self.addCleanup(topology.__exit__, None, None, None)
|
||||
for name in ("_is_cuda", "_is_hip", "_use_aiter"):
|
||||
scope = patch.object(topk_module, name, False)
|
||||
scope.start()
|
||||
self.addCleanup(scope.stop)
|
||||
|
||||
def _model_runner(self, *, device, model):
|
||||
runner = ModelRunner.__new__(ModelRunner)
|
||||
runner.device = device
|
||||
runner.attn_backend = _attention_backend()
|
||||
runner.hisparse_coordinator = None
|
||||
runner.device_timer = None
|
||||
runner.is_draft_worker = False
|
||||
runner.spec_algorithm = SpeculativeAlgorithm.NONE
|
||||
runner.capture_tail_hooks = []
|
||||
runner.canary_manager = None
|
||||
runner.model = model
|
||||
return runner
|
||||
|
||||
def _debug_runner(self):
|
||||
model = _RoutingModel(proxy_output=True)
|
||||
model_runner = self._model_runner(device="cuda", model=model)
|
||||
runner = DecodeCudaGraphRunner.__new__(DecodeCudaGraphRunner)
|
||||
runner.model_runner = model_runner
|
||||
runner.captured_req_width = 1
|
||||
runner.ragged_verify_mode = False
|
||||
runner.enable_pdmux = False
|
||||
runner.pp_size = 1
|
||||
runner.dllm_uses_input_embeds = False
|
||||
runner.in_graph_metadata_prep_done = None
|
||||
runner.is_dllm = False
|
||||
static = _batch(4, 4)
|
||||
runner.attn_backend = captured_backend = _attention_backend()
|
||||
runner.capture_prepare = lambda *args, **kwargs: (
|
||||
static,
|
||||
captured_backend,
|
||||
None,
|
||||
)
|
||||
runner.tbo_plugin = SimpleNamespace(
|
||||
capture_one_batch_size=lambda *args, **kw: None
|
||||
)
|
||||
runner.deepep_adapter = SimpleNamespace(capture=lambda **kwargs: None)
|
||||
stream = _CpuStream()
|
||||
runner.device_module = SimpleNamespace(
|
||||
Stream=_CpuStream,
|
||||
current_stream=lambda *args: stream,
|
||||
synchronize=lambda: None,
|
||||
is_current_stream_capturing=lambda: False,
|
||||
)
|
||||
runner.backend = BreakableCudaGraphBackend(runner, debug_eager=True)
|
||||
runner.can_run_graph = lambda batch: True
|
||||
|
||||
def load_batch(batch, pp_proxy_tensors):
|
||||
# Substitute only the accelerator input copies/shape selection.
|
||||
static.num_token_non_padded.copy_(batch.global_num_token_non_padded)
|
||||
runner.raw_bs = runner.raw_num_token = batch.batch_size
|
||||
runner.bs = 4
|
||||
runner._replay_graph_key = runner._make_graph_key(4)
|
||||
|
||||
runner.load_batch = load_batch
|
||||
model_runner.decode_cuda_graph_runner = runner
|
||||
for target, name, replacement in (
|
||||
(torch.cuda, "CUDAGraph", _NoDeviceGraph),
|
||||
(breakable_graph, "get_device_module", lambda: runner.device_module),
|
||||
(breakable_backend, "graph_pool_capture_scope", nullcontext),
|
||||
(breakable_backend, "graph_pool_replay_scope", nullcontext),
|
||||
):
|
||||
scope = patch.object(target, name, replacement)
|
||||
scope.start()
|
||||
self.addCleanup(scope.stop)
|
||||
with envs.SGLANG_ENABLE_GRAPH_POOL_PRECARVE.override(False):
|
||||
runner.capture_one_shape(4, model.forward)
|
||||
self.assertEqual(
|
||||
len(runner.backend._graphs[runner._make_graph_key(4)]._break_fns), 1
|
||||
)
|
||||
return model_runner, model, static, captured_backend
|
||||
|
||||
def _cpu_runner(self):
|
||||
model = _RoutingModel(proxy_output=False)
|
||||
model_runner = self._model_runner(device="cpu", model=model)
|
||||
runner = CPUGraphRunner.__new__(CPUGraphRunner)
|
||||
runner.model_runner = model_runner
|
||||
runner.is_encoder_decoder = False
|
||||
runner.capture_hidden_mode = CaptureHiddenMode.NULL
|
||||
runner.capture_bs = [4]
|
||||
runner.captured_req_width = 1
|
||||
runner.seq_len_fill_value = 1
|
||||
runner.graphs = {4: model.forward}
|
||||
runner.graphs_cross = {}
|
||||
static = _batch(4, 4)
|
||||
runner.captured_forward_batches = {4: static}
|
||||
runner.can_run_graph = lambda batch: True
|
||||
model_runner.decode_cuda_graph_runner = runner
|
||||
return model_runner, model, static, model_runner.attn_backend
|
||||
|
||||
def _capture_cpu_runner(self):
|
||||
model_runner, model, _, backend = self._cpu_runner()
|
||||
runner = model_runner.decode_cuda_graph_runner
|
||||
runner.capture_bs = [1, 4]
|
||||
runner.capture_forward_mode = ForwardMode.DECODE
|
||||
runner.captured_forward_batches = {}
|
||||
runner.captured_forward_batches_cross = {}
|
||||
runner.input_ids = torch.arange(4)
|
||||
runner.req_pool_indices = torch.arange(4)
|
||||
runner.seq_lens = torch.ones(4, dtype=torch.int32)
|
||||
runner.out_cache_loc = torch.arange(4)
|
||||
runner.positions = torch.arange(4)
|
||||
runner.mrope_positions = torch.zeros((3, 4), dtype=torch.int64)
|
||||
runner.num_token_non_padded = torch.zeros(1, dtype=torch.int64)
|
||||
model_runner.kv_index_translator = SimpleNamespace(
|
||||
bind_runner_slots=lambda batch: None
|
||||
)
|
||||
backend.init_forward_metadata_capture_cpu_graph = lambda *args: None
|
||||
for size in reversed(runner.capture_bs):
|
||||
runner.graphs[size], _ = runner.capture_one_batch_size(
|
||||
size, model.forward, skip_cross_attention=True
|
||||
)
|
||||
return runner, model, backend
|
||||
|
||||
def test_cpu_capture_without_tracking_keeps_live_routes(self):
|
||||
"""Padded CPU replays keep every live route when count tracking is disabled."""
|
||||
with get_parallel().override(
|
||||
tp_size=1,
|
||||
moe_ep_size=1,
|
||||
attn_tp_size=1,
|
||||
tp_group=SimpleNamespace(barrier=lambda: None, world_size=1),
|
||||
):
|
||||
self.assertFalse(enable_num_token_non_padded())
|
||||
runner, model, backend = self._capture_cpu_runner()
|
||||
for size in (3, 4, 2, 1):
|
||||
with self.subTest(size=size):
|
||||
incoming = _batch(size, size, prepared=False)
|
||||
incoming.global_num_token_non_padded = None
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=backend), forward_batch=incoming
|
||||
):
|
||||
eager = model.forward(
|
||||
incoming.input_ids, incoming.positions, incoming
|
||||
)
|
||||
output = runner.execute(incoming)
|
||||
torch.testing.assert_close(
|
||||
output.next_token_logits, eager.next_token_logits
|
||||
)
|
||||
prepared, bound, _, ids = model.seen[-1]
|
||||
self.assertIsNone(bound)
|
||||
self.assertTrue((ids >= 0).all())
|
||||
self.assertEqual(prepared.batch_size, 4 if size > 1 else 1)
|
||||
self.assertFalse(has_forward_context())
|
||||
for captured in runner.captured_forward_batches.values():
|
||||
self.assertIsNone(captured.num_token_non_padded)
|
||||
|
||||
def _assert_replay(self, model_runner, model, static, backend):
|
||||
persistent_bound = static.num_token_non_padded
|
||||
for live in (3, 1, 0):
|
||||
incoming = _batch(live, live, prepared=False)
|
||||
model_runner._forward_raw(incoming, None)
|
||||
batch, bound, active_backend, ids = model.seen[-1]
|
||||
self.assertIs(batch, static)
|
||||
self.assertIs(bound, persistent_bound)
|
||||
self.assertEqual(int(bound), live)
|
||||
self.assertIs(active_backend, backend)
|
||||
self.assertEqual(ids[:live].tolist(), [[0, 1]] * live)
|
||||
self.assertEqual(ids[live:].tolist(), [[-1, -1]] * (4 - live))
|
||||
self.assertIsNone(incoming.num_token_non_padded)
|
||||
self.assertFalse(has_forward_context())
|
||||
|
||||
def test_debug_saved_forward_rebinds_static_batch(self):
|
||||
self._assert_replay(*self._debug_runner())
|
||||
|
||||
def test_cpu_replay_rebinds_actual_prepared_batch(self):
|
||||
self._assert_replay(*self._cpu_runner())
|
||||
|
||||
def _assert_failure_restores_parent(self, model_runner, model, static, backend):
|
||||
parent = _batch(1, 1)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=object()), forward_batch=parent
|
||||
):
|
||||
previous = get_forward_context()
|
||||
model.fail = True
|
||||
with self.assertRaisesRegex(RuntimeError, "replayed model failed"):
|
||||
model_runner._forward_raw(_batch(3, 3, prepared=False), None)
|
||||
self.assertIs(model.seen[-1][0], static)
|
||||
self.assertIs(model.seen[-1][1], static.num_token_non_padded)
|
||||
self.assertIs(model.seen[-1][2], backend)
|
||||
self.assertIs(get_forward_context(), previous)
|
||||
self.assertIs(
|
||||
previous.moe_num_token_non_padded(), parent.num_token_non_padded
|
||||
)
|
||||
self.assertFalse(has_forward_context())
|
||||
|
||||
def test_debug_saved_forward_failure_restores_parent(self):
|
||||
self._assert_failure_restores_parent(*self._debug_runner())
|
||||
|
||||
def test_cpu_replay_failure_restores_parent(self):
|
||||
self._assert_failure_restores_parent(*self._cpu_runner())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -22,12 +22,13 @@ from sglang.test.ci.ci_register import register_cpu_ci
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
|
||||
import sglang.srt.models.deepseek_v4_replay_tail as replay_tail
|
||||
from sglang.srt.layers.moe import utils as moe_utils
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -37,6 +38,15 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
compute_local_num_token_non_padded_cpu,
|
||||
enable_num_token_non_padded,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.runner.row_bucket_graph_runner import (
|
||||
RowBucketGraphRunner,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_flags, get_parallel
|
||||
from sglang.test.communicator_patch import patch_communicator
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -254,5 +264,185 @@ class TestEnableNumTokenNonPadded(CustomTestCase):
|
||||
self.assertIs(enable_num_token_non_padded(), expected)
|
||||
|
||||
|
||||
class TestRowBucketMoeBound(CustomTestCase):
|
||||
def test_capture_owns_the_moe_bound_and_restores_the_outer_context(self):
|
||||
"""A bucket's recorded routing must retain its own mutable count buffer."""
|
||||
for active_context in (False, True):
|
||||
with (
|
||||
self.subTest(active_context=active_context),
|
||||
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
|
||||
sparse_moe_input("local"),
|
||||
):
|
||||
outer_batch = _forward_batch(sharded=False, local=3, glob=3)
|
||||
outer = ForwardContext(attn_backend=None)
|
||||
scope = (
|
||||
forward_context(outer, forward_batch=outer_batch)
|
||||
if active_context
|
||||
else nullcontext()
|
||||
)
|
||||
observed = []
|
||||
|
||||
def body(inputs, capture_batch):
|
||||
if active_context:
|
||||
bound = get_forward_context().moe_num_token_non_padded()
|
||||
self.assertIs(bound, capture_batch.num_token_non_padded)
|
||||
observed.append(bound)
|
||||
else:
|
||||
self.assertFalse(has_forward_context())
|
||||
return inputs
|
||||
|
||||
def record(body, rows, count, batch, start, free_before):
|
||||
body()
|
||||
return SimpleNamespace(batch=(), num_token_non_padded=count)
|
||||
|
||||
runner = RowBucketGraphRunner(row_step=8, max_rows=8, label="test")
|
||||
with (
|
||||
scope,
|
||||
patch.object(torch.cuda, "mem_get_info", return_value=(1, 2)),
|
||||
patch.object(runner, "_record", side_effect=record),
|
||||
):
|
||||
graph = runner._capture(
|
||||
8,
|
||||
3,
|
||||
[torch.zeros(8, 1)],
|
||||
outer_batch,
|
||||
body,
|
||||
lambda batch, inputs: nullcontext(),
|
||||
)
|
||||
graph.num_token_non_padded.fill_(7)
|
||||
outer_batch.num_token_non_padded.fill_(1)
|
||||
if active_context:
|
||||
self.assertEqual(int(observed[0]), 7)
|
||||
self.assertIs(
|
||||
get_forward_context().moe_num_token_non_padded(),
|
||||
outer_batch.num_token_non_padded,
|
||||
)
|
||||
self.assertFalse(has_forward_context())
|
||||
|
||||
|
||||
class TestDecoderReplayTailMoeBound(CustomTestCase):
|
||||
# A row graph replaces LOCAL only, so even equal-valued or aliased GLOBAL
|
||||
# buffers cannot establish that the captured getter reads its own count.
|
||||
_CASES = (
|
||||
("local", False, 3, 7, False, True),
|
||||
("global", True, 3, 7, False, False),
|
||||
("equal_global", True, 3, 3, False, False),
|
||||
("aliased_local", False, 3, 3, True, False),
|
||||
("aliased_global", True, 3, 3, True, False),
|
||||
("unbounded_global", True, 3, None, False, True),
|
||||
("absent_local", True, None, 7, False, True),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _batch_and_tail(sharded, local, glob, alias):
|
||||
batch = _forward_batch(
|
||||
sharded=sharded, local=local, glob=glob, forward_mode=ForwardMode.EXTEND
|
||||
)
|
||||
batch.input_ids = torch.arange(4)
|
||||
if alias:
|
||||
batch.global_num_token_non_padded = batch.num_token_non_padded
|
||||
tail = SimpleNamespace(
|
||||
positions=torch.arange(3), cp_metadata=None, rows=lambda t: t[:3]
|
||||
)
|
||||
return batch, tail
|
||||
|
||||
def test_eager_prefill_declines_a_separate_moe_count_buffer(self):
|
||||
for label, sharded, local, glob, alias, expected in self._CASES:
|
||||
with (
|
||||
self.subTest(bound=label),
|
||||
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
|
||||
sparse_moe_input("ffn"),
|
||||
):
|
||||
batch, tail = self._batch_and_tail(sharded, local, glob, alias)
|
||||
original_global = batch.global_num_token_non_padded
|
||||
tail_layers = replay_tail.DecoderReplayTail.__new__(
|
||||
replay_tail.DecoderReplayTail
|
||||
)
|
||||
tail_layers.runner = SimpleNamespace(
|
||||
ready=Mock(), warm=True, bucket_rows=lambda n: 8
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
replay_tail, "get_is_capture_mode", return_value=False
|
||||
),
|
||||
patch.object(
|
||||
replay_tail,
|
||||
"get_global_expert_distribution_recorder",
|
||||
return_value=SimpleNamespace(recording=False),
|
||||
),
|
||||
patch(
|
||||
"sglang.srt.layers.attention.dsv4.unified_kv_gate.is_unified_kv_triton",
|
||||
return_value=False,
|
||||
),
|
||||
):
|
||||
use_graph, dp_rows = tail_layers.plan(
|
||||
tail,
|
||||
forward_batch=batch,
|
||||
num_tokens=4,
|
||||
tail_in_graph=False,
|
||||
late_dp_counts=None,
|
||||
)
|
||||
self.assertEqual(use_graph, expected)
|
||||
self.assertIsNone(dp_rows)
|
||||
self.assertIs(batch.global_num_token_non_padded, original_global)
|
||||
|
||||
def test_prefill_graph_break_keeps_a_separate_moe_count_eager(self):
|
||||
for label, sharded, local, glob, alias, expected in self._CASES:
|
||||
with (
|
||||
self.subTest(bound=label),
|
||||
get_parallel().override(attn_dp_size=1, attn_cp_size=1),
|
||||
sparse_moe_input("ffn"),
|
||||
):
|
||||
batch, tail = self._batch_and_tail(sharded, local, glob, alias)
|
||||
original_global = batch.global_num_token_non_padded
|
||||
output = torch.tensor([11])
|
||||
tail_layers = SimpleNamespace(
|
||||
runner=SimpleNamespace(warm=True, bucket_rows=lambda n: 8),
|
||||
_can_capture_moe_bound=(
|
||||
replay_tail.DecoderReplayTail._can_capture_moe_bound
|
||||
),
|
||||
run=Mock(return_value=(output, None, [])),
|
||||
_run_layers_eagerly=Mock(return_value=(output, None, [])),
|
||||
_bridge_outputs=lambda rows, outputs: tuple(outputs),
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
forward_metadata=SimpleNamespace(late_layer_tail=tail),
|
||||
exit_late_layer_tail=Mock(),
|
||||
)
|
||||
with (
|
||||
patch.object(replay_tail, "get_attn_backend", return_value=backend),
|
||||
patch.object(
|
||||
replay_tail,
|
||||
"get_tc_piecewise_forward_context",
|
||||
return_value=SimpleNamespace(forward_batch=batch),
|
||||
),
|
||||
patch.object(
|
||||
replay_tail,
|
||||
"_enter_eager_tail_in_prefill_graph",
|
||||
return_value=(None, batch),
|
||||
),
|
||||
get_flags().dp.override(capturing_prefill_graph=False),
|
||||
):
|
||||
outputs = replay_tail._tail_layers_in_prefill_graph(
|
||||
tail_layers,
|
||||
lambda tensors: tensors,
|
||||
False,
|
||||
4,
|
||||
None,
|
||||
False,
|
||||
[torch.zeros(4, 1)],
|
||||
torch.arange(4),
|
||||
batch.input_ids,
|
||||
batch.input_ids,
|
||||
)
|
||||
self.assertIs(outputs[0], output)
|
||||
self.assertEqual(tail_layers.run.call_count, expected)
|
||||
self.assertEqual(
|
||||
tail_layers._run_layers_eagerly.call_count, not expected
|
||||
)
|
||||
backend.exit_late_layer_tail.assert_called_once_with(None, batch)
|
||||
self.assertIs(batch.global_num_token_non_padded, original_global)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -44,6 +44,11 @@ DRAFT_SUFFIX = re.compile(r"(NextN|MTP|DSpark|DFlash|Standalone)$")
|
||||
|
||||
def _target_of(arch: str, archs: dict):
|
||||
"""The arch a draft entry class drafts for, if it is named after it."""
|
||||
# Bailing shares its NextN entry class across generations. Its loader
|
||||
# fusion hooks belong to V3; legacy checkpoints return a fused count of 0
|
||||
# (covered by TestBailingMoeV3Gate in test_shared_experts_fusion_gates.py).
|
||||
if arch == "BailingMoeForCausalLMNextN":
|
||||
return "BailingMoeV3ForCausalLM"
|
||||
match = DRAFT_SUFFIX.search(arch)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
@@ -11,7 +11,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.models.longcat_flash import LongcatFlashRouter # noqa: E402
|
||||
from sglang.srt.models.longcat_flash import ( # noqa: E402
|
||||
LongcatFlashMoE,
|
||||
LongcatFlashRouter,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_exec, get_parallel # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
||||
|
||||
@@ -25,6 +29,37 @@ def _longcat_config(hidden_size, n_routed_experts, *, router_bias=False):
|
||||
|
||||
|
||||
class TestLongcatFlashRouterHpcGemm(CustomTestCase):
|
||||
def test_zero_experts_reject_placement_before_loading_expert_weights(self):
|
||||
"""Identity expert IDs cannot index routed-only placement tables."""
|
||||
config = SimpleNamespace(
|
||||
routed_scaling_factor=1.0,
|
||||
n_routed_experts=4,
|
||||
moe_topk=2,
|
||||
zero_expert_num=2,
|
||||
zero_expert_type="identity",
|
||||
rounter_params_dtype="float32",
|
||||
hidden_act="silu",
|
||||
)
|
||||
with get_context().override_server_args(model_path="dummy", device="cpu"):
|
||||
for algorithm in ("static", "dynamic", "fake", "lp"):
|
||||
with (
|
||||
self.subTest(algorithm=algorithm),
|
||||
get_exec().moe.override(ep_dispatch_algorithm=algorithm),
|
||||
self.assertRaisesRegex(ValueError, "LongCat zero experts"),
|
||||
):
|
||||
LongcatFlashMoE(config=config, layer_id=0)
|
||||
|
||||
with (
|
||||
get_exec().moe.override(ep_dispatch_algorithm=None),
|
||||
get_parallel().override(tp_size=1),
|
||||
patch(
|
||||
"sglang.srt.models.longcat_flash.LongcatFlashRouter",
|
||||
side_effect=RuntimeError("placement admitted"),
|
||||
),
|
||||
self.assertRaisesRegex(RuntimeError, "placement admitted"),
|
||||
):
|
||||
LongcatFlashMoE(config=config, layer_id=0)
|
||||
|
||||
def _assert_dispatches_to_hpc_gemm(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user