[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)
Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
co-authored by
Aditya Suresh
parent
533020b500
commit
299a75ac5d
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
|
||||
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
|
||||
|
||||
m.def("ggml_moe_get_block_size(int type) -> int");
|
||||
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
|
||||
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
|
||||
|
||||
/*
|
||||
* From csrc/mamba
|
||||
|
||||
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
|
||||
const int nrows_y,
|
||||
const int nrows_dst,
|
||||
const int top_k) {
|
||||
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
|
||||
|
||||
const int blocks_per_row_x = ncols_x / qk;
|
||||
const int blocks_per_col_y = nrows_y / QK8_1;
|
||||
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
|
||||
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
|
||||
}
|
||||
|
||||
const int exp_idx = expert_ids[blockIdx.y];
|
||||
if (exp_idx > 255 || exp_idx < 0) return;
|
||||
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
|
||||
if (exp_idx < 0) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < mmq_x; j += nwarps) {
|
||||
const int col_dst = token_offs[j / nwarps];
|
||||
if (col_dst >= ncols_dst) continue;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
|
||||
const auto row_dst = row_dst_0 + threadIdx.x + i;
|
||||
if (row_dst < nrows_dst) {
|
||||
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (exp_idx > 255) return;
|
||||
|
||||
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
|
||||
const block_q8_1* y = (const block_q8_1*)(vy);
|
||||
|
||||
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
|
||||
if (row >= nrows) {
|
||||
return;
|
||||
}
|
||||
if (expert < 0) {
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const int blocks_per_row = ncols / qk;
|
||||
const int blocks_per_warp = vdr * WARP_SIZE / qi;
|
||||
|
||||
@@ -383,14 +383,17 @@ __global__ void Marlin(
|
||||
|
||||
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
|
||||
int parallel = num_tokens_past_padded / moe_block_size;
|
||||
int num_valid_blocks = parallel;
|
||||
if constexpr (kIsEP) {
|
||||
for (int i = 0; i < parallel; i++) {
|
||||
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
|
||||
}
|
||||
// Every warp counts the same inactive blocks without a serial per-thread scan.
|
||||
int num_invalid_blocks = 0;
|
||||
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
|
||||
num_invalid_blocks += expert_ids_ptr[i] == -1;
|
||||
}
|
||||
int num_invalid_blocks = parallel - num_valid_blocks;
|
||||
parallel = num_valid_blocks;
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset /= 2) {
|
||||
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
|
||||
}
|
||||
parallel -= num_invalid_blocks;
|
||||
if (parallel == 0) return;
|
||||
|
||||
int k_tiles = prob_k / 16 / thread_k_blocks;
|
||||
int n_tiles = prob_n / 16 / thread_n_blocks;
|
||||
@@ -518,23 +521,18 @@ __global__ void Marlin(
|
||||
if (par_id >= parallel) return;
|
||||
|
||||
old_expert_id = expert_id;
|
||||
if constexpr (kIsEP) {
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
skip_count--;
|
||||
if (num_invalid_blocks > 0) {
|
||||
int skip_count = block_id == -1 ? par_id : 0;
|
||||
block_id++;
|
||||
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
|
||||
expert_id = expert_ids_ptr[i];
|
||||
if (expert_id != -1) {
|
||||
if (skip_count == 0) {
|
||||
block_id = i;
|
||||
break;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
expert_id = expert_ids_ptr[block_id];
|
||||
skip_count--;
|
||||
};
|
||||
}
|
||||
} else {
|
||||
block_id = par_id;
|
||||
|
||||
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
|
||||
bool is_ep) {
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
int32_t neg_count = 0;
|
||||
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
// Padding and nonlocal routes are both inactive, independently of EP.
|
||||
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
|
||||
|
||||
for (int e = tid; e <= num_experts; e += stride) {
|
||||
int32_t offset;
|
||||
|
||||
@@ -60,6 +60,16 @@ register_kernel(
|
||||
)
|
||||
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.shuffle_weighted_sum",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
|
||||
capabilities=CUDA_ONLY,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def moe_align_block_size(
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
|
||||
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
|
||||
use_int4_w4a16: tl.constexpr,
|
||||
use_int8_w8a16: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Implements the fused computation for a Mixture of Experts (MOE) using
|
||||
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
|
||||
token_mask = offs_token < num_valid_tokens
|
||||
|
||||
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
|
||||
if filter_expert and off_experts == -1:
|
||||
# -----------------------------------------------------------
|
||||
# Write back zeros to the output when the expert is not
|
||||
# in the current expert parallel rank.
|
||||
if off_experts == -1:
|
||||
# Padding and nonlocal routes both use -1, including without EP.
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -377,12 +374,10 @@ def fused_moe_kernel(
|
||||
per_channel_quant: tl.constexpr,
|
||||
even_Ks: tl.constexpr,
|
||||
c_sorted: tl.constexpr,
|
||||
filter_expert: tl.constexpr,
|
||||
swap_ab: tl.constexpr,
|
||||
FUSE_ADD_TO_OUTPUT: tl.constexpr,
|
||||
MASK_OUTPUT: tl.constexpr,
|
||||
FUSE_SUM_ALL_REDUCE: tl.constexpr,
|
||||
LORA_PRESERVE_BASE: tl.constexpr,
|
||||
ROUTER_TOPK: tl.constexpr,
|
||||
FUSE_SWIGLU: tl.constexpr = False,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
@@ -450,16 +445,10 @@ def fused_moe_kernel(
|
||||
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
|
||||
off_experts = off_experts_i32.to(tl.int64)
|
||||
|
||||
if filter_expert and off_experts == -1:
|
||||
if FUSE_SWIGLU:
|
||||
# C is the half-width post-activation buffer here. Rows owned by a
|
||||
# filtered expert are never read (the down-GEMM CTA for this block
|
||||
# early-exits before loading A), and an N-wide zero store would run
|
||||
# past the row end into a neighboring token's data.
|
||||
return
|
||||
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
|
||||
# Write zeros only when this kernel owns the full output; the experimental LoRA
|
||||
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
|
||||
if off_experts == -1:
|
||||
# The down GEMM never reads inactive sorted or half-width intermediates.
|
||||
# Add/reduce outputs already hold their initial value and must stay intact.
|
||||
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
|
||||
write_zeros_to_output(
|
||||
c_ptr,
|
||||
stride_cm,
|
||||
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
|
||||
lora_preserve_base: bool = False,
|
||||
fuse_swiglu: bool = False,
|
||||
) -> None:
|
||||
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
|
||||
assert topk_weights.stride(1) == 1
|
||||
assert sorted_token_ids.stride(0) == 1
|
||||
|
||||
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
|
||||
use_int4_w4a16=use_int4_w4a16,
|
||||
use_int8_w8a16=use_int8_w8a16,
|
||||
even_Ks=even_Ks,
|
||||
filter_expert=filter_expert,
|
||||
**config,
|
||||
)
|
||||
|
||||
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
|
||||
per_channel_quant=per_channel_quant,
|
||||
even_Ks=even_Ks,
|
||||
c_sorted=c_sorted,
|
||||
filter_expert=filter_expert,
|
||||
swap_ab=swap_ab,
|
||||
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
|
||||
MASK_OUTPUT=mask_output,
|
||||
LORA_PRESERVE_BASE=lora_preserve_base,
|
||||
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
|
||||
ROUTER_TOPK=router_topk,
|
||||
FUSE_SWIGLU=fuse_swiglu,
|
||||
|
||||
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
|
||||
num_tokens,
|
||||
stride_m,
|
||||
has_expert_map: tl.constexpr,
|
||||
is_ep: tl.constexpr,
|
||||
has_topk_ids: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
size: tl.constexpr,
|
||||
routed_scaling_factor: tl.constexpr,
|
||||
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
|
||||
if routed_scaling_factor != 1.0:
|
||||
b_val = b_val * routed_scaling_factor
|
||||
if has_expert_map:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
|
||||
mapped_id = tl.load(
|
||||
expert_map_ptr + tl.maximum(id_val, 0),
|
||||
mask=m_mask & (id_val >= 0),
|
||||
other=-1,
|
||||
)
|
||||
expert_mask = mapped_id >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
elif is_ep:
|
||||
elif has_topk_ids:
|
||||
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
|
||||
expert_mask = id_val >= 0
|
||||
b_val = tl.where(expert_mask, b_val, 0.0)
|
||||
a_vec = tl.load(
|
||||
a_base + n * size,
|
||||
mask=mask & expert_mask[:, None],
|
||||
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
|
||||
Shape: (num_tokens, top_k).
|
||||
outputs: Optional pre-allocated output tensor.
|
||||
Shape: (num_tokens, hidden_size).
|
||||
topk_ids: Optional indices of the top-k experts. Used when
|
||||
`expert_map` is provided. Shape: (num_tokens, top_k).
|
||||
topk_ids: Optional indices of the top-k experts. Negative ids always
|
||||
skip the corresponding input load. Shape: (num_tokens, top_k).
|
||||
expert_map: Optional mapping for Expert Parallelism. A value < 0
|
||||
indicates an invalid token/expert pair that will be skipped.
|
||||
|
||||
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
|
||||
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
|
||||
|
||||
# Keep is_ep as a compatible host argument; route validity follows ids.
|
||||
num_tokens, top_k, size = inputs.shape
|
||||
output_shape = (num_tokens, size)
|
||||
if outputs is None:
|
||||
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
|
||||
num_tokens,
|
||||
top_k * size,
|
||||
expert_map is not None,
|
||||
is_ep,
|
||||
topk_ids is not None,
|
||||
top_k,
|
||||
size,
|
||||
1.0 if routed_scaling_factor is None else routed_scaling_factor,
|
||||
|
||||
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
|
||||
use_fp32_reduce: bool = False,
|
||||
is_zp_float: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Project routes, preserving inactive rows in a caller-provided output.
|
||||
|
||||
Inactive rows are zero when this function allocates the output.
|
||||
"""
|
||||
device = a.device
|
||||
|
||||
# Allocate output if not provided
|
||||
if c_or_none is not None:
|
||||
c = c_or_none
|
||||
else:
|
||||
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
|
||||
|
||||
# Early return for zero-size M
|
||||
if size_m == 0:
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Reduce routed rows while skipping inactive permutation entries."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
|
||||
def _shuffle_weighted_sum_kernel(
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
width: tl.constexpr,
|
||||
rows,
|
||||
topk: tl.constexpr,
|
||||
input_stride: tl.constexpr,
|
||||
output_stride: tl.constexpr,
|
||||
weight_stride0,
|
||||
weight_stride1,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
token = tl.program_id(0)
|
||||
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
total = tl.full((BLOCK,), 0, tl.float32)
|
||||
for slot in range(topk):
|
||||
row = tl.load(permutation + token * topk + slot).to(tl.int64)
|
||||
active = (row >= 0) & (row < rows)
|
||||
values = tl.load(
|
||||
inputs + row * input_stride + columns,
|
||||
mask=active & (columns < width),
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
factor = tl.load(
|
||||
weights + token * weight_stride0 + slot * weight_stride1,
|
||||
mask=active,
|
||||
other=0,
|
||||
).to(tl.float32)
|
||||
total += values * factor
|
||||
tl.store(
|
||||
output + token.to(tl.int64) * output_stride + columns,
|
||||
total,
|
||||
mask=columns < width,
|
||||
)
|
||||
|
||||
|
||||
def shuffle_weighted_sum(
|
||||
inputs: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
permutation: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
) -> None:
|
||||
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
|
||||
assert inputs.ndim == output.ndim == weights.ndim == 2
|
||||
assert inputs.shape[1] == output.shape[1]
|
||||
assert inputs.stride(1) == output.stride(1) == 1
|
||||
assert weights.shape[0] == output.shape[0]
|
||||
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
|
||||
if output.numel() == 0:
|
||||
return
|
||||
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
|
||||
inputs,
|
||||
output,
|
||||
permutation,
|
||||
weights,
|
||||
output.shape[1],
|
||||
inputs.shape[0],
|
||||
weights.shape[1],
|
||||
inputs.stride(0),
|
||||
output.stride(0),
|
||||
weights.stride(0),
|
||||
weights.stride(1),
|
||||
BLOCK=256,
|
||||
)
|
||||
@@ -125,6 +125,10 @@ class _StageExecutor:
|
||||
|
||||
# handling DP attention
|
||||
forward_batch: ForwardBatch = inputs["forward_batch"]
|
||||
self._child_ctx = replace(
|
||||
child_ctx if child_ctx is not None else get_forward_context(),
|
||||
moe_num_token_non_padded=forward_batch.moe_num_token_non_padded,
|
||||
)
|
||||
self._global_dp_buffer_len = forward_batch.global_dp_buffer_len
|
||||
self._local_dp_buffer_len = forward_batch.tbo_padded_len
|
||||
self._global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
|
||||
@@ -39,13 +39,16 @@ class ExpertLocationDispatchInfo:
|
||||
rank_invariant: bool = False
|
||||
|
||||
@classmethod
|
||||
def init_new(cls, layer_id: int):
|
||||
def init_new(cls, layer_id: int, *, is_draft: bool = False):
|
||||
if is_draft:
|
||||
return None
|
||||
ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm
|
||||
expert_location_metadata = get_global_expert_location_metadata()
|
||||
assert expert_location_metadata is not None
|
||||
|
||||
if ep_dispatch_algorithm is None:
|
||||
return None
|
||||
expert_location_metadata = get_global_expert_location_metadata()
|
||||
assert expert_location_metadata is not None, (
|
||||
"Expert dispatch requires placement metadata"
|
||||
)
|
||||
|
||||
return cls(
|
||||
ep_dispatch_algorithm=ep_dispatch_algorithm,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutput,
|
||||
_zero_topk_weights_padded_region,
|
||||
capture_routed_experts_if_allowed,
|
||||
select_experts,
|
||||
)
|
||||
@@ -59,6 +59,8 @@ def fused_topk_npu(
|
||||
)
|
||||
|
||||
if renormalize:
|
||||
from sgl_kernel_npu.norm.l1_norm import l1_norm
|
||||
|
||||
topk_weights = l1_norm(
|
||||
topk_weights
|
||||
if topk_config.num_shared_topk_slots == 0
|
||||
@@ -109,9 +111,44 @@ def fused_topk_npu(
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
|
||||
if expert_location_dispatch_info is not None:
|
||||
topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info)
|
||||
get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids)
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
|
||||
return _finalize_topk_npu(
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
router_logits=router_logits,
|
||||
topk_config=topk_config,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
layer_id=layer_id,
|
||||
)
|
||||
|
||||
|
||||
def _finalize_topk_npu(
|
||||
*,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
router_logits,
|
||||
topk_config,
|
||||
num_token_non_padded,
|
||||
expert_location_dispatch_info,
|
||||
layer_id,
|
||||
):
|
||||
num_shared = topk_config.num_shared_topk_slots
|
||||
routed_ids = topk_ids[:, :-num_shared] if num_shared else topk_ids
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, routed_ids)
|
||||
routed_ids = topk_ids_logical_to_physical(routed_ids, expert_location_dispatch_info)
|
||||
topk_ids = (
|
||||
torch.cat([routed_ids, topk_ids[:, -num_shared:]], dim=-1)
|
||||
if num_shared
|
||||
else routed_ids
|
||||
)
|
||||
# NPU dispatch retains valid IDs; duplicating a padded row's routes onto
|
||||
# expert zero can exceed per-expert capacity in fused dispatch backends.
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
recorder_ids = routed_ids
|
||||
if num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_ids = routed_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
get_global_expert_distribution_recorder().on_select_experts(topk_ids=recorder_ids)
|
||||
return StandardTopKOutput(topk_weights, topk_ids, router_logits)
|
||||
|
||||
@@ -1209,10 +1209,6 @@ class AiterAttnBackend(AttentionBackend):
|
||||
bs, forward_batch.seq_lens, forward_batch.req_pool_indices, max_kv_len
|
||||
)
|
||||
|
||||
# Build cu_seqlens_q from this batch's extend lengths. The standard
|
||||
# prefill metadata path leaves self.qo_indptr unset (qo_indptr=None in
|
||||
# ForwardMetadata), so relying on it corrupts multi-sequence batches
|
||||
# (only bs=1 happens to work).
|
||||
cu_seqlens_q = torch.zeros(bs + 1, dtype=torch.int32, device=q.device)
|
||||
cu_seqlens_q[1:] = torch.cumsum(
|
||||
forward_batch.extend_seq_lens.to(torch.int32), dim=0
|
||||
@@ -2598,7 +2594,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
self.indices_updater_prefill.kv_indptr,
|
||||
self.indices_updater_prefill.kv_indices,
|
||||
None,
|
||||
self.indices_updater_prefill.qo_indptr[: bs + 1],
|
||||
None,
|
||||
max(forward_batch.extend_seq_lens_cpu),
|
||||
forward_batch.seq_lens_cpu.max().item(),
|
||||
@@ -4423,7 +4419,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
self.k_scale.reshape(1), # Q is cast at unit scale.
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
@@ -4499,7 +4495,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fp8_q_descale.reshape(1),
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
cu_k,
|
||||
self.forward_metadata.max_q_len,
|
||||
int(self.forward_metadata.max_kv_len),
|
||||
@@ -4538,8 +4534,8 @@ class AiterAttnBackend(AttentionBackend):
|
||||
fp8_q_descale.reshape(1),
|
||||
k_descale.reshape(1),
|
||||
v_descale.reshape(1),
|
||||
self.qo_indptr[:bs0],
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
self.forward_metadata.max_q_len,
|
||||
self.forward_metadata.max_q_len,
|
||||
softmax_scale=layer.scaling,
|
||||
@@ -4628,7 +4624,7 @@ class AiterAttnBackend(AttentionBackend):
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_cache,
|
||||
v_cache,
|
||||
self.qo_indptr[:bs0],
|
||||
self.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_arg,
|
||||
page_table,
|
||||
self.forward_metadata.max_q_len,
|
||||
|
||||
@@ -94,13 +94,13 @@ def forward_extend_vectorized_5d(
|
||||
kv_indices_lin = torch.arange(
|
||||
total_tokens, dtype=torch.int32, device=k_lin.device
|
||||
)
|
||||
kv_indptr_lin = backend.qo_indptr[:bs0]
|
||||
kv_indptr_lin = backend.forward_metadata.qo_indptr[:bs0]
|
||||
max_q = int(backend.forward_metadata.max_q_len)
|
||||
o = mha_batch_prefill_func(
|
||||
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_lin,
|
||||
v_lin,
|
||||
backend.qo_indptr[:bs0],
|
||||
backend.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_lin,
|
||||
kv_indices_lin,
|
||||
max_q,
|
||||
@@ -187,7 +187,7 @@ def forward_extend_vectorized_5d(
|
||||
q_local.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
k_lin,
|
||||
v_lin,
|
||||
backend.qo_indptr[:bs0],
|
||||
backend.forward_metadata.qo_indptr[:bs0],
|
||||
kv_indptr_lin,
|
||||
kv_indices_lin,
|
||||
max_q,
|
||||
|
||||
@@ -10,7 +10,6 @@ from sglang.srt.utils import is_cuda
|
||||
_is_cuda = is_cuda()
|
||||
if _is_cuda:
|
||||
from sgl_kernel import (
|
||||
apply_shuffle_mul_sum,
|
||||
es_fp8_blockwise_scaled_grouped_mm,
|
||||
es_sm100_mxfp8_blockscaled_grouped_mm,
|
||||
es_sm100_mxfp8_blockscaled_grouped_quant,
|
||||
@@ -23,6 +22,7 @@ if _is_cuda:
|
||||
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
|
||||
shuffle_rows_with_scales,
|
||||
)
|
||||
from sglang.kernels.ops.moe.shuffle_weighted_sum import shuffle_weighted_sum
|
||||
|
||||
|
||||
def cutlass_fused_experts_fp8(
|
||||
@@ -142,8 +142,9 @@ def cutlass_fused_experts_fp8(
|
||||
topk = topk_ids.size(1)
|
||||
device = a.device
|
||||
|
||||
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
# prepare_moe_input writes only live routes; gathers still span the tail.
|
||||
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
|
||||
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
|
||||
|
||||
if use_mxfp8:
|
||||
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
|
||||
@@ -345,5 +346,5 @@ def cutlass_fused_experts_fp8(
|
||||
if output is None:
|
||||
output = torch.empty((m, k), device=device, dtype=out_dtype)
|
||||
|
||||
apply_shuffle_mul_sum(c2, output, c_map, topk_weights.to(out_dtype))
|
||||
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
|
||||
return output
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda, is_cuda_alike
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -128,8 +127,8 @@ def cutlass_w4a8_moe(
|
||||
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
|
||||
|
||||
device = a.device
|
||||
if get_parallel().moe_ep_size > 1:
|
||||
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
|
||||
# Both graph padding and nonlocal routes use -1, including without EP.
|
||||
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
|
||||
|
||||
src2dst = cutlass_w4_run_moe_ep_preproess(
|
||||
topk_ids,
|
||||
@@ -144,7 +143,11 @@ def cutlass_w4a8_moe(
|
||||
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
|
||||
if a1_scale is None:
|
||||
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
|
||||
per_tensor_absmax_fp8(a, a1_scale)
|
||||
# Inactive rows must not change the quantization of live expert inputs.
|
||||
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
|
||||
scale_input = a.masked_fill(inactive_rows, 0.0)
|
||||
per_tensor_absmax_fp8(scale_input, a1_scale)
|
||||
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
|
||||
|
||||
pre_reorder_for_cutlass_moe(
|
||||
a,
|
||||
|
||||
@@ -305,15 +305,14 @@ def fused_marlin_moe(
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
# Marlin skips masked expert rows, so their shared cache must start at zero.
|
||||
intermediate_cache13 = torch.zeros(
|
||||
(M * topk_ids.shape[1] * max(gemm1_n, K),),
|
||||
intermediate_cache13 = torch.empty(
|
||||
(M * topk * max(gemm1_n, K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * gemm1_n]
|
||||
intermediate_cache1 = intermediate_cache1.view(-1, gemm1_n)
|
||||
intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K]
|
||||
intermediate_cache3 = intermediate_cache13[: M * topk * K]
|
||||
intermediate_cache3 = intermediate_cache3.view(-1, K)
|
||||
|
||||
use_atomic_add = (
|
||||
@@ -381,9 +380,9 @@ def fused_marlin_moe(
|
||||
else:
|
||||
raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}")
|
||||
|
||||
if expert_map is not None:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
# Gate-up is dead after activation, so its storage can hold the down output.
|
||||
# Skipped down routes must start at zero before the top-k reduction.
|
||||
intermediate_cache3.zero_()
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
intermediate_cache3,
|
||||
|
||||
@@ -74,6 +74,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_global_dwdp_manager,
|
||||
get_parallel,
|
||||
get_server_args,
|
||||
@@ -367,7 +368,8 @@ class FusedMoE(torch.nn.Module):
|
||||
self.params_dtype = params_dtype
|
||||
self.layer_name = prefix
|
||||
self.layer_id = layer_id
|
||||
self.uses_expert_location = uses_expert_location
|
||||
self.is_draft = get_flags().moe.in_speculative_scope
|
||||
self.uses_expert_location = uses_expert_location and not self.is_draft
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.num_experts = num_experts
|
||||
@@ -1559,6 +1561,14 @@ class FusedMoE(torch.nn.Module):
|
||||
|
||||
return forward_fuseep(self, hidden_states, topk_output)
|
||||
if is_in_tc_piecewise_cuda_graph():
|
||||
if (
|
||||
TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
and topk_output.expert_location_dispatch_info is not None
|
||||
):
|
||||
# The opaque logits-only op cannot carry placement metadata.
|
||||
return self.forward_impl(
|
||||
hidden_states, topk_output, pre_quant_input=pre_quant_input
|
||||
)
|
||||
if TopKOutputChecker.format_is_standard(topk_output):
|
||||
return moe_forward_piecewise_cuda_graph_impl(
|
||||
hidden_states,
|
||||
@@ -1872,7 +1882,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
|
||||
) -> torch.Tensor:
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=TopKConfig(
|
||||
top_k=top_k,
|
||||
topk_group=topk_group,
|
||||
@@ -1880,6 +1890,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
|
||||
correction_bias=correction_bias,
|
||||
renormalize=renormalize,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture,
|
||||
layer_id=layer_id,
|
||||
),
|
||||
)
|
||||
forward_context = get_tc_piecewise_forward_context()
|
||||
|
||||
@@ -21,11 +21,10 @@ from sglang.srt.layers.moe.topk import (
|
||||
_mask_topk_ids_padded_region,
|
||||
_zero_topk_weights_padded_region,
|
||||
remap_topk_for_per_rank_shared_slots,
|
||||
resolve_routing_inputs,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
|
||||
from sglang.srt.runtime_context import get_flags, get_model
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -46,9 +45,13 @@ class HashTopK(nn.Module):
|
||||
routed_scaling_factor=1.5,
|
||||
apply_routed_scaling_factor_on_output=False,
|
||||
layer_id: Optional[int] = None,
|
||||
is_draft: Optional[bool] = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
self.is_draft = (
|
||||
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
|
||||
)
|
||||
|
||||
self.shared_experts = get_model().shared_experts
|
||||
self.enable_waterfill = self.shared_experts.use_waterfill
|
||||
@@ -75,6 +78,9 @@ class HashTopK(nn.Module):
|
||||
"with fused shared experts; pass --disable-shared-experts-fusion."
|
||||
)
|
||||
|
||||
def routing_inputs(self):
|
||||
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
|
||||
|
||||
@property
|
||||
def num_shared_topk_slots(self) -> int:
|
||||
return self.shared_experts.num_shared_topk_slots
|
||||
@@ -100,7 +106,8 @@ class HashTopK(nn.Module):
|
||||
self, device: torch.device, *, layer_id: Optional[int] = None
|
||||
):
|
||||
topk = self.topk - self.num_shared_topk_slots
|
||||
if layer_id is not None:
|
||||
if not self.is_draft:
|
||||
layer_id = self.layer_id
|
||||
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
|
||||
|
||||
lplb_solver = get_global_lplb_solver(layer_id)
|
||||
@@ -226,6 +233,12 @@ class HashTopK(nn.Module):
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
):
|
||||
num_token_non_padded, expert_location_dispatch_info = resolve_routing_inputs(
|
||||
layer_id=self.layer_id,
|
||||
is_draft=self.is_draft,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
assert input_ids.shape[0] == hidden_states.shape[0] == router_logits.shape[0], (
|
||||
f"{input_ids.shape=} {hidden_states.shape=} {router_logits.shape=}"
|
||||
)
|
||||
@@ -252,6 +265,9 @@ class HashTopK(nn.Module):
|
||||
topk_weights = topk_weights * self.routed_scaling_factor
|
||||
|
||||
num_shared_topk_slots = self.num_shared_topk_slots
|
||||
routed_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots] if num_shared_topk_slots else topk_ids
|
||||
)
|
||||
log2phy_prob = None
|
||||
if (
|
||||
expert_location_dispatch_info is not None
|
||||
@@ -263,18 +279,13 @@ class HashTopK(nn.Module):
|
||||
|
||||
lplb_solver = get_global_lplb_solver(self.layer_id)
|
||||
if lplb_solver is not None:
|
||||
log2phy_prob = lplb_solver.solve(topk_ids)
|
||||
log2phy_prob = lplb_solver.solve(routed_ids)
|
||||
|
||||
recorder_topk_ids = None
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
routed_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
recorder_topk_ids = routed_cols if num_shared_topk_slots else None
|
||||
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
|
||||
# The trailing columns are placeholders; the remap appends the real ones.
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
recorder_topk_ids = routed_cols
|
||||
|
||||
num_physical_routed_experts = (
|
||||
expert_location_dispatch_info.num_physical_experts
|
||||
if expert_location_dispatch_info is not None
|
||||
@@ -292,11 +303,13 @@ class HashTopK(nn.Module):
|
||||
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
elif num_shared_topk_slots:
|
||||
topk_ids = torch.cat(
|
||||
[routed_cols, topk_ids[:, -num_shared_topk_slots:]], dim=-1
|
||||
)
|
||||
if is_hip():
|
||||
else:
|
||||
topk_ids = routed_cols
|
||||
if _is_hip or _is_npu:
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
else:
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
@@ -304,6 +317,11 @@ class HashTopK(nn.Module):
|
||||
_mask_topk_ids_padded_region(recorder_topk_ids, num_token_non_padded)
|
||||
if recorder_topk_ids is None:
|
||||
recorder_topk_ids = topk_ids
|
||||
if _is_npu and num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_topk_ids = recorder_topk_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_topk_ids
|
||||
)
|
||||
@@ -311,7 +329,7 @@ class HashTopK(nn.Module):
|
||||
topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits
|
||||
)
|
||||
topk_output = self._apply_waterfill(topk_output, hidden_states.shape[0])
|
||||
if is_hip():
|
||||
if _is_hip or _is_npu:
|
||||
_zero_topk_weights_padded_region(
|
||||
topk_output.topk_weights, num_token_non_padded
|
||||
)
|
||||
|
||||
@@ -23,7 +23,6 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.moe.mega_moe_sm90 import (
|
||||
@@ -263,16 +262,6 @@ def _run_mega_routed(
|
||||
if num_tokens > 0:
|
||||
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
|
||||
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
|
||||
num_token_non_padded = (
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
)
|
||||
if (
|
||||
not moe.is_hash
|
||||
and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None
|
||||
@@ -281,15 +270,11 @@ def _run_mega_routed(
|
||||
moe,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
)
|
||||
else:
|
||||
topk_output = moe.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
topk_ids = topk_output.topk_ids
|
||||
|
||||
@@ -14,7 +14,6 @@ import torch
|
||||
|
||||
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
@@ -307,25 +306,16 @@ def _run_mega_routed(
|
||||
topk_output = moe.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.num_token_non_padded
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
x_in = hidden_states
|
||||
topk_ids = topk_output.topk_ids.to(torch.int32)
|
||||
topk_weights = topk_output.topk_weights.to(torch.float32)
|
||||
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
|
||||
num_token_non_padded, _ = moe.topk.routing_inputs()
|
||||
if num_token_non_padded is not None:
|
||||
_fill_padded_rows(
|
||||
topk_ids,
|
||||
forward_batch.num_token_non_padded,
|
||||
num_token_non_padded,
|
||||
experts,
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_model,
|
||||
@@ -110,6 +111,10 @@ from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import (
|
||||
cpu_has_amx_support,
|
||||
@@ -279,6 +284,8 @@ class TopKConfig:
|
||||
# Draft-side MoE blocks set this False so they never write the target's
|
||||
# process-global routed-experts capture buffer.
|
||||
allow_routed_experts_capture: bool = True
|
||||
# Lazy materialization still needs the producer's solver and capture slot.
|
||||
layer_id: Optional[int] = None
|
||||
_correction_bias_dtype_cache: Optional[torch.Tensor] = field(
|
||||
default=None, init=False, repr=False, compare=False
|
||||
)
|
||||
@@ -393,10 +400,10 @@ class TritonKernelTopKOutput(NamedTuple):
|
||||
|
||||
|
||||
class BypassedTopKOutput(NamedTuple):
|
||||
"""Bypassed top-k output format."""
|
||||
"""Lazy routing that can be materialized or consumed by an in-kernel router."""
|
||||
|
||||
hidden_states: torch.Tensor
|
||||
router_logits: torch.Tensor
|
||||
raw_router_logits: torch.Tensor
|
||||
topk_config: TopKConfig
|
||||
num_token_non_padded: Optional[torch.Tensor] = None
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None
|
||||
@@ -405,14 +412,23 @@ class BypassedTopKOutput(NamedTuple):
|
||||
def format(self) -> TopKOutputFormat:
|
||||
return TopKOutputFormat.BYPASSED
|
||||
|
||||
@property
|
||||
def router_logits(self) -> torch.Tensor:
|
||||
if self.expert_location_dispatch_info is not None:
|
||||
raise NotImplementedError(
|
||||
"Expert placement remapping requires materialized TopK routing; "
|
||||
"this MoE kernel routes directly from logits."
|
||||
)
|
||||
return self.raw_router_logits
|
||||
|
||||
def to_standard(self, layer_id: Optional[int] = None) -> StandardTopKOutput:
|
||||
"""Materialize routing tensors. Used by MoE kernels that need explicit
|
||||
topk_ids / topk_weights rather than doing routing internally."""
|
||||
return select_experts(
|
||||
hidden_states=self.hidden_states,
|
||||
router_logits=self.router_logits,
|
||||
router_logits=self.raw_router_logits,
|
||||
topk_config=self.topk_config,
|
||||
layer_id=layer_id,
|
||||
layer_id=self.topk_config.layer_id if layer_id is None else layer_id,
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
expert_location_dispatch_info=self.expert_location_dispatch_info,
|
||||
)
|
||||
@@ -630,6 +646,24 @@ def _can_bypass_trtllm_routing(config: TopKConfig) -> bool:
|
||||
# -------------------------------- TopK ---------------------------------------
|
||||
|
||||
|
||||
def resolve_routing_inputs(
|
||||
*,
|
||||
layer_id: int,
|
||||
is_draft: bool,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
):
|
||||
if num_token_non_padded is None and has_forward_context():
|
||||
get_bound = get_forward_context().moe_num_token_non_padded
|
||||
if get_bound is not None:
|
||||
num_token_non_padded = get_bound()
|
||||
if is_draft:
|
||||
expert_location_dispatch_info = None
|
||||
elif expert_location_dispatch_info is None:
|
||||
expert_location_dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id)
|
||||
return num_token_non_padded, expert_location_dispatch_info
|
||||
|
||||
|
||||
class TopK(BaseFusedOp):
|
||||
"""
|
||||
Parameters:
|
||||
@@ -649,6 +683,7 @@ class TopK(BaseFusedOp):
|
||||
top_k: int,
|
||||
*,
|
||||
layer_id: int,
|
||||
is_draft: Optional[bool] = None,
|
||||
use_grouped_topk: bool = False,
|
||||
topk_group: Optional[int] = None,
|
||||
num_expert_group: Optional[int] = None,
|
||||
@@ -674,6 +709,9 @@ class TopK(BaseFusedOp):
|
||||
assert num_expert_group is not None and topk_group is not None
|
||||
|
||||
self.layer_id = layer_id
|
||||
self.is_draft = (
|
||||
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
|
||||
)
|
||||
self._runner_backend_bound = False
|
||||
|
||||
shared_experts = get_model().shared_experts
|
||||
@@ -686,6 +724,7 @@ class TopK(BaseFusedOp):
|
||||
self.is_fp4_experts = is_fp4_experts
|
||||
self.topk_config = TopKConfig(
|
||||
top_k=top_k + num_shared_slots,
|
||||
layer_id=layer_id,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
@@ -703,7 +742,21 @@ class TopK(BaseFusedOp):
|
||||
scoring_func=scoring_func,
|
||||
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
|
||||
fused_gate_packed_ids=fused_gate_packed_ids,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture,
|
||||
allow_routed_experts_capture=allow_routed_experts_capture
|
||||
and not self.is_draft,
|
||||
)
|
||||
|
||||
def routing_inputs(self):
|
||||
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
|
||||
|
||||
def _resolve_routing_inputs(
|
||||
self, num_token_non_padded, expert_location_dispatch_info
|
||||
):
|
||||
return resolve_routing_inputs(
|
||||
layer_id=self.layer_id,
|
||||
is_draft=self.is_draft,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
)
|
||||
|
||||
def bind_runner_backend(
|
||||
@@ -778,6 +831,11 @@ class TopK(BaseFusedOp):
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
router_logits_partials: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if router_logits_partials is not None:
|
||||
rocm_router_reduce_partials(router_logits_partials, router_logits)
|
||||
self.topk_config.torch_native = True
|
||||
@@ -805,6 +863,12 @@ class TopK(BaseFusedOp):
|
||||
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
|
||||
fixed-order sum is the logits; router_logits is then a buffer the fused gate
|
||||
fills, and any other reader first reduces the partials into it."""
|
||||
explicit_dispatch_info = expert_location_dispatch_info is not None
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if dynamic_expert_bias is not None:
|
||||
output_format = TopKOutputFormat.STANDARD
|
||||
elif self.topk_config.output_format is not None:
|
||||
@@ -814,6 +878,17 @@ class TopK(BaseFusedOp):
|
||||
"TopK requires the owning experts' runner backend or an explicit output format."
|
||||
)
|
||||
|
||||
requires_remapping = expert_location_dispatch_info is not None and (
|
||||
explicit_dispatch_info
|
||||
or is_expert_remapping_enabled()
|
||||
or expert_location_dispatch_info.ep_dispatch_algorithm in ("fake", "lp")
|
||||
)
|
||||
if output_format == TopKOutputFormat.TRITON_KERNEL and requires_remapping:
|
||||
raise NotImplementedError(
|
||||
"Expert placement remapping requires a MoE runner with standard TopK routing; "
|
||||
f"got {output_format.name}."
|
||||
)
|
||||
|
||||
if (
|
||||
router_logits_partials is not None
|
||||
and output_format != TopKOutputFormat.STANDARD
|
||||
@@ -833,6 +908,7 @@ class TopK(BaseFusedOp):
|
||||
router_logits,
|
||||
self.topk_config.top_k,
|
||||
sm_first=not self.topk_config.renormalize,
|
||||
n_rows=num_token_non_padded,
|
||||
)
|
||||
return TritonKernelTopKOutput(
|
||||
a_ragged_metadata,
|
||||
@@ -844,10 +920,12 @@ class TopK(BaseFusedOp):
|
||||
elif output_format == TopKOutputFormat.BYPASSED:
|
||||
return BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
raw_router_logits=router_logits,
|
||||
topk_config=self.topk_config,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
expert_location_dispatch_info=(
|
||||
expert_location_dispatch_info if requires_remapping else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.topk_config.torch_native = False
|
||||
@@ -875,6 +953,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
topk_output = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
layer_id=self.layer_id,
|
||||
@@ -895,6 +978,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
if dynamic_expert_bias is not None:
|
||||
self.topk_config.torch_native = False
|
||||
return select_experts(
|
||||
@@ -923,12 +1011,13 @@ class TopK(BaseFusedOp):
|
||||
) -> TopKOutput:
|
||||
"""Return an empty topk output for a rank with zero tokens this forward.
|
||||
|
||||
When ``layer_id`` is provided and the active dispatch algorithm is LP,
|
||||
also calls ``LPLBSolver.solve(empty)`` so that this rank participates
|
||||
in the EP all-reduce. Without this, an empty rank would skip the
|
||||
collective and deadlock under DP-attention.
|
||||
Target layers call their owning ``LPLBSolver.solve(empty)`` so this
|
||||
rank participates in the EP all-reduce even without tokens. Draft
|
||||
layers skip the target solver. The optional ``layer_id`` is retained
|
||||
for caller compatibility; the router's owning layer determines the solver.
|
||||
"""
|
||||
if layer_id is not None:
|
||||
if not self.is_draft:
|
||||
layer_id = self.layer_id
|
||||
# Skip the full ExpertLocationDispatchInfo allocation — we only
|
||||
# need the per-layer solver to participate in the EP all-reduce.
|
||||
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
|
||||
@@ -972,6 +1061,11 @@ class TopK(BaseFusedOp):
|
||||
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
|
||||
dynamic_expert_bias: Optional[torch.Tensor] = None,
|
||||
) -> TopKOutput:
|
||||
num_token_non_padded, expert_location_dispatch_info = (
|
||||
self._resolve_routing_inputs(
|
||||
num_token_non_padded, expert_location_dispatch_info
|
||||
)
|
||||
)
|
||||
self.topk_config.torch_native = True
|
||||
# [NOTE] XPU device support for topk kernels
|
||||
# - support 'topk_softmax' and 'topk_sigmoid'
|
||||
@@ -1816,6 +1910,12 @@ def _mask_topk_ids_padded_region(
|
||||
) -> None:
|
||||
if num_token_non_padded is None:
|
||||
return
|
||||
if not topk_ids.is_cuda:
|
||||
if _is_npu:
|
||||
return
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
topk_ids.masked_fill_((rows >= num_token_non_padded)[:, None], fill_value)
|
||||
return
|
||||
# TODO: let the kernel support other dtypes
|
||||
if _is_cuda and topk_ids.dtype == torch.int32 and fill_value == -1:
|
||||
mask_topk_ids(topk_ids, num_token_non_padded)
|
||||
@@ -1834,6 +1934,10 @@ def _zero_topk_weights_padded_region(
|
||||
):
|
||||
if num_token_non_padded is None:
|
||||
return
|
||||
if not topk_weights.is_cuda:
|
||||
rows = torch.arange(topk_weights.shape[0], device=topk_weights.device)
|
||||
topk_weights.masked_fill_((rows >= num_token_non_padded)[:, None], 0.0)
|
||||
return
|
||||
if _can_fuse_padded_region(topk_weights):
|
||||
_fill_padded_rows(topk_weights, num_token_non_padded, 0.0)
|
||||
return
|
||||
@@ -2420,12 +2524,27 @@ def _post_process_topk_ids(
|
||||
|
||||
lplb_solver = get_global_lplb_solver(layer_id)
|
||||
if lplb_solver is not None:
|
||||
log2phy_prob = lplb_solver.solve(topk_ids)
|
||||
lp_routed_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots]
|
||||
if router_wrote_shared_cols
|
||||
else topk_ids
|
||||
)
|
||||
log2phy_prob = lplb_solver.solve(lp_routed_ids)
|
||||
|
||||
if log2phy_prob is not None:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
if router_wrote_shared_cols:
|
||||
shared_cols = topk_ids[:, -num_shared_topk_slots:]
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots],
|
||||
expert_location_dispatch_info,
|
||||
log2phy_prob,
|
||||
)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = topk_ids[:, :-num_shared_topk_slots]
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info, log2phy_prob
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
elif router_wrote_shared_cols:
|
||||
# Shared IDs are outside EPLB's routed-expert table for both global
|
||||
@@ -2506,6 +2625,21 @@ def _post_process_topk_ids(
|
||||
# That final pass re-zeros after any shared-expert append/remap, so a
|
||||
# second zeroing here would be redundant (zeroing is idempotent).
|
||||
|
||||
else:
|
||||
if num_shared_topk_slots > 0:
|
||||
shared_cols = topk_ids[:, -num_shared_topk_slots:]
|
||||
routed_cols = topk_ids_logical_to_physical(
|
||||
topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info
|
||||
)
|
||||
_mask_topk_ids_padded_region(routed_cols, num_token_non_padded)
|
||||
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
|
||||
recorder_topk_ids = routed_cols
|
||||
else:
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info
|
||||
)
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
|
||||
|
||||
if recorder_topk_ids is None and not recorder_was_fused:
|
||||
recorder_topk_ids = (
|
||||
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
|
||||
@@ -2598,6 +2732,14 @@ def _post_process_topk_ids(
|
||||
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
|
||||
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
|
||||
|
||||
if _is_npu:
|
||||
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
|
||||
if recorder_topk_ids is not None and num_token_non_padded is not None:
|
||||
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
|
||||
recorder_topk_ids = recorder_topk_ids.masked_fill(
|
||||
(rows >= num_token_non_padded)[:, None], -1
|
||||
)
|
||||
|
||||
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
|
||||
# Shared-expert append/remap can introduce non-zero weights after the
|
||||
# initial HIP padding mask above. Ensure padded tokens leave this helper
|
||||
@@ -2750,10 +2892,6 @@ def select_experts(
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
)
|
||||
elif torch_native and custom_routing_function is None:
|
||||
assert num_token_non_padded is None, (
|
||||
"num_token_non_padded is not yet supported in fused_topk_native"
|
||||
)
|
||||
assert expert_location_dispatch_info is None
|
||||
fold_scale_after_routing = True
|
||||
topk_weights, topk_ids = fused_topk_native(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -387,6 +387,9 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
from sglang.kernels.ops.activation.activation import silu_and_mul
|
||||
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
_map_topk_ids_to_local_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
|
||||
|
||||
hidden_states = hidden_states.contiguous()
|
||||
@@ -398,7 +401,7 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
|
||||
local_topk_ids = topk_ids.to(torch.int32)
|
||||
if expert_map is not None:
|
||||
local_topk_ids = expert_map[local_topk_ids]
|
||||
local_topk_ids = _map_topk_ids_to_local_experts(local_topk_ids, expert_map)
|
||||
local_topk_ids = local_topk_ids.contiguous()
|
||||
topk_weights = topk_scores.to(torch.float32).contiguous()
|
||||
|
||||
@@ -430,13 +433,13 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache13 = torch.empty(
|
||||
(num_tokens * topk * max(gemm1_n, size_k),),
|
||||
cache13 = torch.zeros(
|
||||
(num_tokens * topk * (gemm1_n + size_k),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
|
||||
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
|
||||
cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k)
|
||||
|
||||
cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
@@ -525,6 +528,7 @@ class MuseSpark1xFp4Experts(nn.Module):
|
||||
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
|
||||
hidden_states, topk_scores, topk_ids
|
||||
)
|
||||
topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0)
|
||||
if self._post_expert_norm is not post_expert_norm:
|
||||
self.set_post_expert_norm(post_expert_norm)
|
||||
if self._post_expert_norm_weight is not None:
|
||||
|
||||
@@ -18,6 +18,9 @@ from sglang.srt.hardware_backend.npu.quantization.moe_methods import (
|
||||
from sglang.srt.hardware_backend.npu.utils import npu_format_cast
|
||||
from sglang.srt.layers.linear import LinearBase
|
||||
from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig
|
||||
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend, get_moe_runner_backend
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
FusedMoEMethodBase,
|
||||
@@ -41,24 +44,22 @@ _is_musa = is_musa()
|
||||
_is_npu = is_npu()
|
||||
|
||||
if _is_cuda:
|
||||
from sgl_kernel import moe_align_block_size, moe_sum
|
||||
from sgl_kernel import moe_sum
|
||||
from sgl_kernel.quantization import (
|
||||
ggml_dequantize,
|
||||
ggml_moe_a8,
|
||||
ggml_moe_a8_vec,
|
||||
ggml_moe_get_block_size,
|
||||
ggml_mul_mat_a8,
|
||||
ggml_mul_mat_vec_a8,
|
||||
)
|
||||
|
||||
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
|
||||
elif _is_musa:
|
||||
from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul
|
||||
from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul
|
||||
from sgl_kernel.quantization import (
|
||||
ggml_dequantize,
|
||||
ggml_moe_a8,
|
||||
ggml_moe_a8_vec,
|
||||
ggml_moe_get_block_size,
|
||||
ggml_mul_mat_a8,
|
||||
ggml_mul_mat_vec_a8,
|
||||
)
|
||||
@@ -180,6 +181,16 @@ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
|
||||
MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
|
||||
|
||||
|
||||
def ggml_moe_get_block_size(qweight_type: int) -> int:
|
||||
# MOE_X in the CUDA/MUSA GGUF MMQ kernels is 4. The released wheel's
|
||||
# Tensor-free metadata operator has no catch-all dispatch registration.
|
||||
return (
|
||||
(8 if _is_hip else 4)
|
||||
if qweight_type in MMQ_QUANT_TYPES and qweight_type != WeightType.Q8_1
|
||||
else 0
|
||||
)
|
||||
|
||||
|
||||
def dequantize_gguf_weight(
|
||||
qweight: torch.Tensor, qweight_type: int, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
@@ -239,6 +250,8 @@ def fused_moe_gguf(
|
||||
return gelu_and_mul(x)
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
inactive = topk_ids < 0
|
||||
x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0)
|
||||
out_hidden_states = torch.empty_like(x)
|
||||
# unless we decent expert reuse we are better off running moe_vec kernel
|
||||
if (
|
||||
@@ -254,6 +267,11 @@ def fused_moe_gguf(
|
||||
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
|
||||
topk_ids, BLOCK_SIZE, E
|
||||
)
|
||||
# Released MMQ kernels inspect one slack block before their bound check.
|
||||
block_starts = (
|
||||
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
|
||||
)
|
||||
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
|
||||
out = ggml_moe_a8(
|
||||
x,
|
||||
w1,
|
||||
@@ -265,6 +283,7 @@ def fused_moe_gguf(
|
||||
top_k,
|
||||
num_tokens,
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
out = act(out)
|
||||
out = ggml_moe_a8(
|
||||
out,
|
||||
@@ -277,8 +296,10 @@ def fused_moe_gguf(
|
||||
1,
|
||||
num_tokens * top_k,
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
valid_weights = topk_weights.masked_fill(inactive, 0.0)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
valid_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
# TODO(FlamingoPg): maybe we can use moe_sum_reduce here?
|
||||
moe_sum(out, out_hidden_states)
|
||||
@@ -287,14 +308,20 @@ def fused_moe_gguf(
|
||||
E, N, _ = w1.shape
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
out = ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
|
||||
# Older vector kernels form weight addresses before checking -1.
|
||||
# They have no per-expert capacity buffer; discard their dummy slots.
|
||||
safe_ids = topk_ids.clamp_min(0)
|
||||
out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
out = act(out)
|
||||
|
||||
out = ggml_moe_a8_vec(
|
||||
out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
|
||||
out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
|
||||
)
|
||||
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
|
||||
valid_weights = topk_weights.masked_fill(inactive, 0.0)
|
||||
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
|
||||
topk_weights.view(num_tokens, top_k, 1)
|
||||
valid_weights.view(num_tokens, top_k, 1)
|
||||
)
|
||||
moe_sum(out, out_hidden_states)
|
||||
else:
|
||||
@@ -305,8 +332,10 @@ def fused_moe_gguf(
|
||||
)
|
||||
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
|
||||
inp = x[tok].reshape((1,) + x.shape[1:])
|
||||
current_hidden_state = None
|
||||
current_hidden_state = torch.zeros_like(inp)
|
||||
for ww, ii in zip(w, idx):
|
||||
if ii < 0:
|
||||
continue
|
||||
expert_up = w1[ii]
|
||||
|
||||
out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
|
||||
@@ -316,10 +345,7 @@ def fused_moe_gguf(
|
||||
current_state = fused_mul_mat_gguf(
|
||||
out, expert_down, qweight_type2
|
||||
).mul_(ww)
|
||||
if current_hidden_state is None:
|
||||
current_hidden_state = current_state
|
||||
else:
|
||||
current_hidden_state.add_(current_state)
|
||||
current_hidden_state.add_(current_state)
|
||||
out_hidden_states[tok] = current_hidden_state
|
||||
return out_hidden_states
|
||||
|
||||
@@ -565,8 +591,6 @@ class GGUFMoEMethod(FusedMoEMethodBase):
|
||||
layer: torch.nn.Module,
|
||||
dispatch_output: StandardDispatchOutput,
|
||||
) -> CombineInput:
|
||||
assert self.fused_experts is None
|
||||
|
||||
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
|
||||
|
||||
assert self.moe_runner_config.activation == "silu", (
|
||||
|
||||
@@ -71,8 +71,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
|
||||
assert runner_config.activation == "silu", "Only SiLU activation is supported."
|
||||
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
|
||||
@@ -119,9 +119,13 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
)
|
||||
|
||||
# Stage 1: Gate/Up (Marlin)
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -165,11 +169,6 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: Down (Marlin)
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
if quant_info.expert_map is not None:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""No-sort B200 decode kernels for Inkling routed MoE LoRA.
|
||||
|
||||
The contract is BF16 CUDA, M<=32, top-k 6, rank 32, up to four slots, TP-local
|
||||
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)``.
|
||||
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)`` or the inactive sentinel ``-1``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -47,7 +47,8 @@ def _direct_gate_expand_kernel(
|
||||
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
offs_r = tl.arange(0, RANK)
|
||||
@@ -110,7 +111,8 @@ def _direct_down_shrink_kernel(
|
||||
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
|
||||
token = route // ROUTER_TOPK
|
||||
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS)
|
||||
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
|
||||
expert = tl.maximum(expert, 0)
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_r = tl.arange(0, RANK)
|
||||
offs_k = tl.arange(0, BLOCK_K)
|
||||
@@ -121,7 +123,7 @@ def _direct_down_shrink_kernel(
|
||||
k_mask = k < INTERMEDIATE_SIZE
|
||||
x = tl.load(
|
||||
activation_ptr + route * stride_xq + k[None, :] * stride_xk,
|
||||
mask=(offs_m[:, None] == 0) & k_mask[None, :],
|
||||
mask=active & (offs_m[:, None] == 0) & k_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
a = tl.load(
|
||||
|
||||
@@ -232,8 +232,8 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
topk_output = dispatch_output.topk_output
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
|
||||
if (
|
||||
topk_ids.ndim != 2
|
||||
or topk_weights.shape != topk_ids.shape
|
||||
@@ -345,7 +345,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
!= lora_info.gate_up_lora_b_weights.shape[-1]
|
||||
)
|
||||
# Allocate and prewarm on main; side streams launch kernels only.
|
||||
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
|
||||
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
|
||||
if not direct_decode:
|
||||
merged_experts_fused_moe_lora_add(
|
||||
output=gate_up_delta,
|
||||
@@ -572,9 +572,13 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
)
|
||||
|
||||
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
|
||||
intermediate_cache1 = torch.empty(
|
||||
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
projection_cache = torch.zeros(
|
||||
(M * topk * (2 * N + K),),
|
||||
device=hidden_states.device,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
|
||||
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
|
||||
intermediate_cache1 = moe_wna16_marlin_gemm(
|
||||
hidden_states,
|
||||
intermediate_cache1,
|
||||
@@ -657,8 +661,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
zero_intermediate=(
|
||||
ep_active
|
||||
and lora_info.experts_shared_outer_loras
|
||||
lora_info.experts_shared_outer_loras
|
||||
and not factored_shared_outer
|
||||
),
|
||||
)
|
||||
@@ -679,11 +682,6 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
|
||||
|
||||
# Stage 3: down (marlin).
|
||||
intermediate_cache3 = torch.empty(
|
||||
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
if ep_active:
|
||||
intermediate_cache3.zero_()
|
||||
|
||||
intermediate_cache3 = moe_wna16_marlin_gemm(
|
||||
intermediate_cache2,
|
||||
@@ -764,7 +762,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
routing_cache=routing_cache,
|
||||
local_expert_offset=0,
|
||||
local_num_experts=E,
|
||||
zero_intermediate=(ep_active and lora_info.experts_shared_outer_loras),
|
||||
zero_intermediate=lora_info.experts_shared_outer_loras,
|
||||
)
|
||||
|
||||
# Never alias hidden_states: the shared sink still reads it. The fused B200
|
||||
@@ -866,9 +864,7 @@ def fused_experts_experimental_sgl_marlin_lora(
|
||||
down_intermediate if down_shrink_done is not None else None
|
||||
),
|
||||
zero_intermediate=(
|
||||
ep_active
|
||||
and lora_info.experts_shared_outer_loras
|
||||
and down_shrink_done is None
|
||||
lora_info.experts_shared_outer_loras and down_shrink_done is None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -835,7 +835,9 @@ class CPUGraphRunner:
|
||||
spec_algorithm=self.model_runner.spec_algorithm,
|
||||
spec_info=spec_info,
|
||||
capture_hidden_mode=self.capture_hidden_mode,
|
||||
num_token_non_padded=self.num_token_non_padded,
|
||||
num_token_non_padded=(
|
||||
self.num_token_non_padded if enable_num_token_non_padded() else None
|
||||
),
|
||||
global_forward_mode=self.capture_forward_mode,
|
||||
)
|
||||
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
|
||||
@@ -849,7 +851,8 @@ class CPUGraphRunner:
|
||||
)
|
||||
with skip_ctx:
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph(
|
||||
bs,
|
||||
@@ -991,11 +994,15 @@ class CPUGraphRunner:
|
||||
with replay_context():
|
||||
with skip_ctx:
|
||||
prepared_forward_batch = self.prepare_replay(forward_batch, skip=skip)
|
||||
output = graphs[prepared_forward_batch.batch_size](
|
||||
prepared_forward_batch.input_ids,
|
||||
prepared_forward_batch.positions,
|
||||
prepared_forward_batch,
|
||||
)
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=prepared_forward_batch,
|
||||
):
|
||||
output = graphs[prepared_forward_batch.batch_size](
|
||||
prepared_forward_batch.input_ids,
|
||||
prepared_forward_batch.positions,
|
||||
prepared_forward_batch,
|
||||
)
|
||||
if forward_batch.batch_size in graphs:
|
||||
return output
|
||||
|
||||
|
||||
@@ -1143,14 +1143,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
batch.extend_input_logprob_token_ids.to(device, non_blocking=True)
|
||||
)
|
||||
|
||||
num_tokens = len(batch.input_ids) if batch.input_ids is not None else 0
|
||||
if enable_num_token_non_padded():
|
||||
ret.global_num_token_non_padded = torch.tensor(
|
||||
num_tokens,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(device),
|
||||
).to(device, non_blocking=True)
|
||||
ret.global_num_token_non_padded_cpu = num_tokens
|
||||
ret._init_num_token_non_padded(
|
||||
device=device, is_draft_worker=model_runner.is_draft_worker
|
||||
)
|
||||
|
||||
ret.init_mlp_sync_metadata(
|
||||
batch, device, is_draft_worker=model_runner.is_draft_worker
|
||||
@@ -1341,6 +1336,26 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
pin_memory=is_pin_memory_available(batch.device),
|
||||
).to(batch.device, non_blocking=True)
|
||||
|
||||
def _init_num_token_non_padded(self, *, device, is_draft_worker: bool) -> None:
|
||||
if self.input_ids is not None:
|
||||
num_tokens = len(self.input_ids)
|
||||
elif (
|
||||
is_draft_worker
|
||||
and self.spec_info is not None
|
||||
and self.forward_mode.is_decode()
|
||||
):
|
||||
assert self.spec_info.num_tokens_per_req > 0
|
||||
num_tokens = self.batch_size * self.spec_info.num_tokens_per_req
|
||||
else:
|
||||
num_tokens = 0
|
||||
if enable_num_token_non_padded():
|
||||
self.global_num_token_non_padded = torch.tensor(
|
||||
num_tokens,
|
||||
dtype=torch.int32,
|
||||
pin_memory=is_pin_memory_available(device),
|
||||
).to(device, non_blocking=True)
|
||||
self.global_num_token_non_padded_cpu = num_tokens
|
||||
|
||||
def set_local_num_token_non_padded(self, *, sharded: bool) -> None:
|
||||
"""Derive the LOCAL num_token_non_padded from the invariant GLOBAL scalar.
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ layer reads at depth via get_forward_context(). The only mandatory field
|
||||
today is attn_backend; pool refs are derived from attn_backend.*
|
||||
(every backend caches them at __init__), so a published ForwardContext
|
||||
is enough to resolve the active pools without a separate global.
|
||||
The active batch lends its MoE bound getter so late preparation and TBO child
|
||||
selection stay visible without retaining a parent tensor snapshot.
|
||||
|
||||
ModelRunner._forward_raw publishes a fresh ForwardContext for the
|
||||
duration of each forward; callers that need a per-call override (PDmux
|
||||
@@ -23,12 +25,15 @@ worker threads ever share a process, migrate to contextvars.ContextVar.
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -38,6 +43,8 @@ class ForwardContext:
|
||||
write time — use dataclasses.replace for per-call overrides."""
|
||||
|
||||
attn_backend: AttentionBackend
|
||||
# Resolve after batch preparation; captured calls retain the static buffer.
|
||||
moe_num_token_non_padded: Optional[Callable[[], Optional[torch.Tensor]]] = None
|
||||
|
||||
|
||||
_current: Optional[ForwardContext] = None
|
||||
@@ -76,7 +83,15 @@ def get_req_to_token_pool() -> ReqToTokenPool:
|
||||
|
||||
|
||||
@contextmanager
|
||||
def forward_context(ctx: ForwardContext):
|
||||
def forward_context(
|
||||
ctx: ForwardContext, *, forward_batch: Optional[ForwardBatch] = None
|
||||
):
|
||||
if forward_batch is not None:
|
||||
ctx = replace(
|
||||
ctx, moe_num_token_non_padded=forward_batch.moe_num_token_non_padded
|
||||
)
|
||||
elif ctx.moe_num_token_non_padded is None and _current is not None:
|
||||
ctx = replace(ctx, moe_num_token_non_padded=_current.moe_num_token_non_padded)
|
||||
prev = set_forward_context(ctx)
|
||||
try:
|
||||
yield
|
||||
|
||||
@@ -113,6 +113,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.graph_memory_usage import (
|
||||
@@ -1965,9 +1966,14 @@ class ModelRunner:
|
||||
if prepare is not None:
|
||||
prepare(forward_batch)
|
||||
if has_forward_context():
|
||||
ctx_mgr = contextlib.nullcontext()
|
||||
ctx_mgr = forward_context(
|
||||
get_forward_context(), forward_batch=forward_batch
|
||||
)
|
||||
else:
|
||||
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
|
||||
ctx_mgr = forward_context(
|
||||
ForwardContext(attn_backend=self.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
with (
|
||||
get_model().override(shared_experts=self.model.shared_experts_config),
|
||||
ctx_mgr,
|
||||
|
||||
@@ -762,7 +762,9 @@ class BaseRunner(ABC):
|
||||
|
||||
torch.get_device_module(mr.device).synchronize()
|
||||
get_parallel().tp_group.barrier()
|
||||
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=mr.attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
with run_ctx or empty_context():
|
||||
run_once()
|
||||
|
||||
|
||||
@@ -1253,7 +1253,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# All setup hooks below read get_attn_backend() (TboForwardBatchPreparer,
|
||||
# DeepEP adapter, …) so they must run inside the same ForwardContext
|
||||
# that wraps the warmup/capture forward.
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens)
|
||||
|
||||
if forward_batch.lora_ids is not None:
|
||||
@@ -1262,60 +1264,67 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
|
||||
|
||||
def run_once():
|
||||
# Backends may gather live pool mappings into their static
|
||||
# metadata here; these nodes run before any model KV reads.
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
self._record_in_graph_metadata_prep_done()
|
||||
|
||||
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
|
||||
None
|
||||
)
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
forward_batch.global_num_tokens_cpu,
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
kwargs = {}
|
||||
if (
|
||||
self.pp_size > 1
|
||||
and "pp_proxy_tensors" in inspect.signature(forward).parameters
|
||||
# Debug replay retains this callback beyond the capture scope.
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
kwargs["pp_proxy_tensors"] = PPProxyTensors(
|
||||
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
|
||||
)
|
||||
if (
|
||||
(
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
if self.dllm_input_preparation is None:
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
|
||||
else:
|
||||
state = self.buffer_registry.get_slot(
|
||||
"dllm_input_preparation_state"
|
||||
).buffer[:num_tokens]
|
||||
kwargs["input_embeds"] = self.dllm_input_preparation(
|
||||
self.model_runner.model, forward_batch.input_ids, state
|
||||
)
|
||||
# Backends may gather live pool mappings into their static
|
||||
# metadata here; these nodes run before any model KV reads.
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
self._record_in_graph_metadata_prep_done()
|
||||
|
||||
out = forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
for capture_hook in self.model_runner.capture_tail_hooks:
|
||||
capture_hook(self, out, forward_batch, num_tokens)
|
||||
return out
|
||||
forward_batch.dp_local_start_pos = (
|
||||
forward_batch.dp_local_num_tokens
|
||||
) = None
|
||||
set_dp_buffer_len(
|
||||
forward_batch.global_dp_buffer_len,
|
||||
num_tokens,
|
||||
forward_batch.dp_padding_mode.is_max_len(),
|
||||
forward_batch.global_num_tokens_cpu,
|
||||
)
|
||||
set_is_extend_in_batch(False)
|
||||
|
||||
kwargs = {}
|
||||
if (
|
||||
self.pp_size > 1
|
||||
and "pp_proxy_tensors" in inspect.signature(forward).parameters
|
||||
):
|
||||
kwargs["pp_proxy_tensors"] = PPProxyTensors(
|
||||
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
|
||||
)
|
||||
if (
|
||||
(
|
||||
self.dllm_uses_input_embeds
|
||||
or (
|
||||
self.model_runner.spec_algorithm.is_dflash_family()
|
||||
and self.model_runner.is_draft_worker
|
||||
)
|
||||
)
|
||||
and "input_embeds" in inspect.signature(forward).parameters
|
||||
and not hasattr(self.model_runner.model, "forward_embed")
|
||||
):
|
||||
if self.dllm_input_preparation is None:
|
||||
kwargs["input_embeds"] = self.buffers.input_embeds[
|
||||
:num_tokens
|
||||
]
|
||||
else:
|
||||
state = self.buffer_registry.get_slot(
|
||||
"dllm_input_preparation_state"
|
||||
).buffer[:num_tokens]
|
||||
kwargs["input_embeds"] = self.dllm_input_preparation(
|
||||
self.model_runner.model, forward_batch.input_ids, state
|
||||
)
|
||||
|
||||
out = forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
**kwargs,
|
||||
)
|
||||
for capture_hook in self.model_runner.capture_tail_hooks:
|
||||
capture_hook(self, out, forward_batch, num_tokens)
|
||||
return out
|
||||
|
||||
self.deepep_adapter.capture(is_extend_in_batch=False)
|
||||
canary_ctx = (
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.model_executor.forward_batch_info import (
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
ForwardContext,
|
||||
forward_context,
|
||||
get_forward_context,
|
||||
get_req_to_token_pool,
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
@@ -259,7 +260,11 @@ class EagerRunner(BaseRunner):
|
||||
|
||||
ctx = device_timer_ctx(model_runner.device_timer, "decode")
|
||||
|
||||
with ctx, pdmux_ctx:
|
||||
with (
|
||||
ctx,
|
||||
pdmux_ctx,
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
return model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
@@ -341,7 +346,10 @@ class EagerRunner(BaseRunner):
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
else "extend"
|
||||
)
|
||||
with device_timer_ctx(model_runner.device_timer, category):
|
||||
with (
|
||||
device_timer_ctx(model_runner.device_timer, category),
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
if cp_active and (
|
||||
not get_parallel().enable_cp_tp_group_sharing
|
||||
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
|
||||
@@ -448,7 +456,10 @@ class EagerRunner(BaseRunner):
|
||||
model_runner.attn_backend.forward_metadata = None
|
||||
|
||||
kwargs = model_runner._pp_kwargs(pp_proxy_tensors)
|
||||
with device_timer_ctx(model_runner.device_timer, "idle"):
|
||||
with (
|
||||
device_timer_ctx(model_runner.device_timer, "idle"),
|
||||
forward_context(get_forward_context(), forward_batch=forward_batch),
|
||||
):
|
||||
return model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
|
||||
@@ -819,7 +819,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
full_graph=self._is_full_backend, raw_num_tokens=raw_num_tokens
|
||||
),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
set_tc_piecewise_forward_context(
|
||||
forward_batch,
|
||||
@@ -953,7 +954,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
set_forward_batch(fb),
|
||||
prefill_graph_scope(full_graph=self._is_full_backend),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend)
|
||||
ForwardContext(attn_backend=self.model_runner.attn_backend),
|
||||
forward_batch=fb,
|
||||
),
|
||||
set_tc_piecewise_forward_context(
|
||||
fb,
|
||||
@@ -1234,7 +1236,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
and stash the returned per-bucket metadata object; otherwise fall
|
||||
back to the generic eager init that BCG/TC_PIECEWISE use today."""
|
||||
attn_backend = self.model_runner.attn_backend
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
if not self.use_captured_attn_metadata:
|
||||
attn_backend.init_forward_metadata(forward_batch)
|
||||
return
|
||||
|
||||
@@ -20,7 +20,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
set_graph_pool_id,
|
||||
)
|
||||
from sglang.srt.distributed.parallel_state import graph_capture
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
forward_context,
|
||||
get_attn_backend,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
BreakableCUDAGraph,
|
||||
BreakableCUDAGraphCapture,
|
||||
@@ -120,7 +125,14 @@ class RowBucketGraphRunner:
|
||||
# The eager breaks read the step's real row count off this view.
|
||||
step_batch = copy.copy(forward_batch)
|
||||
step_batch.global_num_token_non_padded_cpu = num_rows
|
||||
with context(step_batch):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=step_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
context(step_batch),
|
||||
):
|
||||
if (rows, graph_key) not in self._graphs:
|
||||
# Largest first, so smaller buckets reuse the pool memory the
|
||||
# larger ones took; a bucket below this step's rows cannot be
|
||||
@@ -168,7 +180,14 @@ class RowBucketGraphRunner:
|
||||
forward_batch.num_token_non_padded, num_rows
|
||||
)
|
||||
capture_batch.num_token_non_padded = num_token_non_padded
|
||||
with batch_scope(capture_batch, inputs):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=capture_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
batch_scope(capture_batch, inputs),
|
||||
):
|
||||
graph = self._record(
|
||||
lambda: body(inputs, capture_batch),
|
||||
rows,
|
||||
|
||||
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -492,10 +491,6 @@ class BailingMoESparseMoeBlock(nn.Module):
|
||||
hidden_states,
|
||||
router_logits,
|
||||
dynamic_expert_bias=dynamic_expert_bias,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -21,7 +21,6 @@ from sglang.srt.configs import KimiLinearConfig
|
||||
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
@@ -832,10 +831,6 @@ class BailingMoE(nn.Module):
|
||||
hidden_states,
|
||||
router_logits,
|
||||
dynamic_expert_bias=dynamic_expert_bias,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
|
||||
@@ -66,7 +66,6 @@ from sglang.srt.distributed import divide
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.amx_utils import PackWeightMethod
|
||||
@@ -120,10 +119,10 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
|
||||
DispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
BypassedTopKOutput,
|
||||
TopK,
|
||||
TopKOutputFormat,
|
||||
fused_shared_expert_weight,
|
||||
is_expert_remapping_enabled,
|
||||
is_routed_scaling_factor_fused_in_topk,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
@@ -1038,7 +1037,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
and not self._enable_a2a_moe
|
||||
and not self._fuse_shared_experts_inside_sbo
|
||||
and not getattr(self, "is_hash", False)
|
||||
and not get_exec().moe.enable_eplb
|
||||
and not is_expert_remapping_enabled()
|
||||
and get_exec().moe.ep_dispatch_algorithm not in ("fake", "lp")
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1067,11 +1067,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids_global=input_ids_global,
|
||||
)
|
||||
|
||||
num_token_non_padded = (
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
|
||||
if use_vision_topk and _is_hip:
|
||||
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
|
||||
@@ -1109,7 +1104,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
gemm_output_zero_allocator,
|
||||
input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
@@ -1121,7 +1115,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids,
|
||||
input_ids_global=input_ids_global,
|
||||
skip_shared_experts=skip_shared_experts,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
use_vision_topk=use_vision_topk,
|
||||
return_moe_output=return_moe_output,
|
||||
leave_output_sum=leave_output_sum,
|
||||
@@ -1137,7 +1130,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
gemm_output_zero_allocator: BumpAllocator = None,
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
@@ -1167,11 +1159,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
has_shared_output = (
|
||||
hidden_states.shape[0] > 0 and self.num_fused_shared_experts == 0
|
||||
)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb and not self.is_nextn
|
||||
else None
|
||||
)
|
||||
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
if _is_hip:
|
||||
@@ -1185,11 +1172,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
|
||||
router_logits_partials = None
|
||||
if use_flashinfer_trtllm_bypass:
|
||||
topk_output = BypassedTopKOutput(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
topk_config=self.topk.topk_config,
|
||||
)
|
||||
topk_output = self.topk.forward_cuda(hidden_states, router_logits)
|
||||
else:
|
||||
topk_kwargs = (
|
||||
{"input_ids": input_ids_global}
|
||||
@@ -1203,14 +1186,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
self,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
# Issued after the router so the main chain stays on the main stream at replay.
|
||||
@@ -1323,7 +1303,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
input_ids: Optional[torch.Tensor] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
skip_shared_experts: bool = False,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
use_vision_topk: bool = False,
|
||||
return_moe_output: bool = False,
|
||||
leave_output_sum: bool = False,
|
||||
@@ -1332,11 +1311,6 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.shared_experts.gate_up_proj
|
||||
):
|
||||
return self.forward_cpu(hidden_states)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb and not self.is_nextn
|
||||
else None
|
||||
)
|
||||
defer_shared = not self.experts.moe_runner_config.inplace
|
||||
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): shared expert is computed on the LOCAL
|
||||
# hidden in the decoder layer (before the dp gather) and added after the
|
||||
@@ -1383,21 +1357,18 @@ class DeepseekV2MoE(nn.Module):
|
||||
self,
|
||||
router_logits,
|
||||
input_ids_global,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
pre_quant_input = None
|
||||
shared_output = None
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
if self._fuse_shared_experts_inside_sbo and not skip_shared_experts:
|
||||
@@ -1591,19 +1562,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
if sbo_overlap_dispatch_flag:
|
||||
@@ -1979,19 +1942,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
|
||||
@@ -28,7 +28,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
forward_context,
|
||||
get_attn_backend,
|
||||
get_forward_context,
|
||||
has_forward_context,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
compute_attention_and_moe_layers,
|
||||
)
|
||||
@@ -163,6 +168,15 @@ class DecoderReplayTail:
|
||||
max_rows=max_rows,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _can_capture_moe_bound(forward_batch: ForwardBatch) -> bool:
|
||||
# The row runner replaces only the local count with a graph-owned buffer.
|
||||
bound = forward_batch.moe_num_token_non_padded()
|
||||
return bound is None or (
|
||||
bound is forward_batch.num_token_non_padded
|
||||
and bound is not forward_batch.global_num_token_non_padded
|
||||
)
|
||||
|
||||
def plan(
|
||||
self,
|
||||
tail: Optional[LateLayerTail],
|
||||
@@ -199,6 +213,7 @@ class DecoderReplayTail:
|
||||
or get_global_expert_distribution_recorder().recording
|
||||
or not self.runner.warm
|
||||
or (tail is not None and tail.cp_metadata is not None)
|
||||
or not self._can_capture_moe_bound(forward_batch)
|
||||
):
|
||||
return False, None
|
||||
if not dp:
|
||||
@@ -414,7 +429,14 @@ class DecoderReplayTail:
|
||||
gathered = input_ids.new_empty((sum(dp_rows), 1))
|
||||
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
|
||||
input_ids_global = gathered.squeeze(-1)
|
||||
with self._break_context(tail_batch):
|
||||
with (
|
||||
(
|
||||
forward_context(get_forward_context(), forward_batch=tail_batch)
|
||||
if has_forward_context()
|
||||
else nullcontext()
|
||||
),
|
||||
self._break_context(tail_batch),
|
||||
):
|
||||
return self._layer_stack(
|
||||
state,
|
||||
positions=positions,
|
||||
@@ -681,6 +703,7 @@ def _tail_layers_in_prefill_graph(
|
||||
if (
|
||||
not get_flags().dp.capturing_prefill_graph
|
||||
and runner.warm
|
||||
and tail_layers._can_capture_moe_bound(forward_batch)
|
||||
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
|
||||
):
|
||||
residual, pre, aux = tail_layers.run(**kwargs)
|
||||
|
||||
@@ -51,7 +51,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers import deep_gemm_wrapper
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
|
||||
@@ -467,14 +466,10 @@ class Dots3MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
final_hidden_states = self.experts(
|
||||
|
||||
@@ -26,7 +26,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -228,10 +227,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -653,10 +652,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -30,7 +30,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
)
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
@@ -377,10 +376,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -34,7 +34,6 @@ from triton.language.extra import libdevice
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -918,10 +917,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -97,6 +97,7 @@ from sglang.srt.model_loader.utils import (
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
get_stream,
|
||||
)
|
||||
@@ -251,6 +252,15 @@ class LongcatFlashMoE(nn.Module):
|
||||
self.top_k = config.moe_topk
|
||||
self.zero_expert_num = config.zero_expert_num
|
||||
self.zero_expert_type = config.zero_expert_type
|
||||
if (
|
||||
self.zero_expert_num > 0
|
||||
and get_exec().moe.ep_dispatch_algorithm is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"LongCat zero experts do not support expert placement remapping. "
|
||||
"Disable EPLB, redundant experts, custom expert placement, and "
|
||||
"--ep-dispatch-algorithm for this model."
|
||||
)
|
||||
|
||||
if config.rounter_params_dtype == "float32":
|
||||
self.rounter_params_dtype = torch.float32
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -559,11 +558,6 @@ class MiMoV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if self._uses_expert_remap
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -595,10 +589,6 @@ class MiMoV2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -628,10 +618,6 @@ class MiMoV2MoE(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
AuxHiddenStateList,
|
||||
@@ -614,10 +613,6 @@ class MiniMaxM2MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(device=hidden_states.device)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.configs.model_config import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -535,10 +534,6 @@ class MiniMaxM3MoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -29,7 +29,6 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import ReLU2
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
|
||||
HybridLinearAttnBackend,
|
||||
@@ -349,7 +348,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if (
|
||||
@@ -364,21 +362,16 @@ class NemotronHMoE(nn.Module):
|
||||
and is_in_breakable_cuda_graph()
|
||||
and is_sm120()
|
||||
):
|
||||
return self._forward_core_prefill_overlap(
|
||||
hidden_states, num_token_non_padded
|
||||
)
|
||||
return self._forward_core_prefill_overlap(hidden_states)
|
||||
if _is_cuda and (
|
||||
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
|
||||
):
|
||||
return self._forward_core_shared_routed_overlap(
|
||||
hidden_states, num_token_non_padded
|
||||
)
|
||||
return self._forward_core_normal(hidden_states, num_token_non_padded)
|
||||
return self._forward_core_shared_routed_overlap(hidden_states)
|
||||
return self._forward_core_normal(hidden_states)
|
||||
|
||||
def _forward_core_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# router_scores: [num_tokens, num_experts]
|
||||
# bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk.
|
||||
@@ -392,12 +385,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -407,7 +394,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core_shared_routed_overlap(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
alt_stream = _get_or_create_alt_stream(self.device_module)
|
||||
|
||||
@@ -427,12 +413,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -444,7 +424,6 @@ class NemotronHMoE(nn.Module):
|
||||
def _forward_core_prefill_overlap(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# A multi-stream graph per hybrid segment creates one internal stream
|
||||
# per executable. Keep each graph linear and launch shared experts on
|
||||
@@ -460,12 +439,6 @@ class NemotronHMoE(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
@@ -509,11 +482,6 @@ class NemotronHMoE(nn.Module):
|
||||
# MoE runner / topk), so final_hidden_states is already scaled.
|
||||
final_hidden_states, shared_output = self._forward_core(
|
||||
hidden_states,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStateAccumulator,
|
||||
@@ -611,14 +610,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -667,16 +658,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.enable_shared_expert_fusion:
|
||||
topk_output = self._append_shared_to_topk_output(
|
||||
|
||||
@@ -28,7 +28,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
@@ -342,20 +341,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if get_exec().moe.enable_eplb
|
||||
else None
|
||||
)
|
||||
|
||||
if hidden_states.shape[0] > 0:
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
)
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
@@ -370,10 +359,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
@@ -408,14 +393,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.moe_num_token_non_padded()
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
assert TopKOutputChecker.format_is_standard(topk_output), (
|
||||
"MegaMoE pre-dispatch consumes raw topk ids/weights; "
|
||||
@@ -454,10 +431,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -13,7 +13,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
@@ -159,10 +158,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -6,7 +6,6 @@ import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -256,10 +255,6 @@ class Step3p5MoEMLP(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
@@ -38,7 +38,6 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
@@ -1000,10 +999,6 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
),
|
||||
)
|
||||
# Apply router scaling factor after renormalization
|
||||
if self.router_scaling_factor != 1.0:
|
||||
@@ -1014,7 +1009,7 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
topk_output.topk_weights = scaled_weights
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
@@ -1051,7 +1046,7 @@ class XllmSparseMoeBlock(nn.Module):
|
||||
if hidden_states.shape[0] == 0:
|
||||
shared_output = None
|
||||
topk_output = self.topk.empty_topk_output(
|
||||
hidden_states.device, layer_id=self.layer_id
|
||||
hidden_states.device,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
else:
|
||||
|
||||
@@ -2,6 +2,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location_dispatch import transform_select_experts_inputs
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
_RENORMALIZE_SUM_EPSILON,
|
||||
StandardTopKOutput,
|
||||
@@ -12,19 +13,10 @@ from sglang.srt.layers.moe.topk import (
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
has_per_rank_fused_shared_slots,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
|
||||
def _scale_fused_shared_weights(weights, num_fused_shared_experts, scaling_factor):
|
||||
# Standard EP replicates the fused shared expert on every rank and all-reduces,
|
||||
# so the shared columns carry a 1/ep_size factor.
|
||||
if num_fused_shared_experts and scaling_factor is not None:
|
||||
weights[:, -num_fused_shared_experts:] *= scaling_factor
|
||||
return weights
|
||||
|
||||
|
||||
def vision_topk(
|
||||
moe,
|
||||
logits,
|
||||
@@ -32,25 +24,35 @@ def vision_topk(
|
||||
num_token_non_padded=None,
|
||||
expert_location_dispatch_info=None,
|
||||
):
|
||||
bound, dispatch_info = moe.topk.routing_inputs()
|
||||
if num_token_non_padded is None:
|
||||
num_token_non_padded = bound
|
||||
if expert_location_dispatch_info is None:
|
||||
expert_location_dispatch_info = dispatch_info
|
||||
config = moe.topk.topk_config
|
||||
logits, correction_bias = transform_select_experts_inputs(
|
||||
logits, moe.gate.e_score_correction_bias, expert_location_dispatch_info
|
||||
)
|
||||
correction_bias_vl = moe.gate.e_score_correction_bias_vl
|
||||
if (
|
||||
expert_location_dispatch_info is not None
|
||||
and expert_location_dispatch_info.ep_dispatch_algorithm == "fake"
|
||||
):
|
||||
correction_bias_vl = torch.zeros_like(correction_bias_vl)
|
||||
num_shared_topk_slots = config.num_shared_topk_slots
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if num_shared_topk_slots and not use_mega_moe:
|
||||
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
|
||||
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
|
||||
"VL routing does not support per-rank fused shared slots"
|
||||
)
|
||||
if is_cuda():
|
||||
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
|
||||
from sglang.srt.layers.moe.utils import get_moe_runner_backend
|
||||
|
||||
# Same admission as _fused_gate_emits_packed_ids: the shared-expert slots
|
||||
# rescaled below would rewrite weights after the router.
|
||||
packed_topk = None
|
||||
if (
|
||||
num_shared_topk_slots == 0
|
||||
and expert_location_dispatch_info is None
|
||||
and not use_mega_moe
|
||||
and get_moe_runner_backend().is_flashinfer_mxfp4()
|
||||
and config.runner_backend is not None
|
||||
and config.runner_backend.is_flashinfer_mxfp4()
|
||||
):
|
||||
packed_topk = torch.empty(
|
||||
(logits.shape[0], config.top_k), dtype=torch.int32, device=logits.device
|
||||
@@ -58,11 +60,11 @@ def vision_topk(
|
||||
|
||||
weights, indices = moe_fused_gate(
|
||||
logits,
|
||||
moe.gate.e_score_correction_bias,
|
||||
correction_bias,
|
||||
topk=config.top_k,
|
||||
scoring_func="sqrtsoftplus",
|
||||
num_fused_shared_experts=num_shared_topk_slots,
|
||||
bias_alt=moe.gate.e_score_correction_bias_vl,
|
||||
bias_alt=correction_bias_vl,
|
||||
input_ids=input_ids,
|
||||
bias_alt_token_id=moe.config.image_token_id,
|
||||
renormalize=config.renormalize and config.top_k > 1,
|
||||
@@ -87,12 +89,12 @@ def vision_topk(
|
||||
return StandardTopKOutput(weights, indices, logits)
|
||||
scores = F.softplus(logits.float()).sqrt()
|
||||
if input_ids is None:
|
||||
bias = moe.gate.e_score_correction_bias
|
||||
bias = correction_bias
|
||||
else:
|
||||
bias = torch.where(
|
||||
(input_ids == moe.config.image_token_id)[:, None],
|
||||
moe.gate.e_score_correction_bias_vl,
|
||||
moe.gate.e_score_correction_bias,
|
||||
correction_bias_vl,
|
||||
correction_bias,
|
||||
)
|
||||
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
|
||||
topk_routed = config.top_k - num_shared_topk_slots
|
||||
@@ -136,28 +138,19 @@ def _finish_vision_topk(
|
||||
):
|
||||
config = moe.topk.topk_config
|
||||
use_mega_moe = get_moe_a2a_backend().is_megamoe()
|
||||
if use_mega_moe:
|
||||
# Use the same EPLB mapping, recording and home-rank shared slots as
|
||||
# ordinary top-k before handing physical expert IDs to MegaMoE.
|
||||
indices, weights, recorder_ids = _post_process_topk_ids(
|
||||
indices,
|
||||
weights,
|
||||
config,
|
||||
logits,
|
||||
moe.layer_id,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
else:
|
||||
weights = _scale_fused_shared_weights(
|
||||
weights,
|
||||
config.num_shared_topk_slots,
|
||||
config.fused_shared_experts_scaling_factor,
|
||||
indices, weights, recorder_ids = _post_process_topk_ids(
|
||||
indices,
|
||||
weights,
|
||||
config,
|
||||
logits,
|
||||
moe.layer_id,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=expert_location_dispatch_info,
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
# Shared-slot remapping can overwrite padded weights; mask last so idle
|
||||
# and partially padded DP ranks never dispatch a phantom shared expert.
|
||||
|
||||
@@ -511,7 +511,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
|
||||
return ret
|
||||
|
||||
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.draft_attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
|
||||
@@ -482,7 +482,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
return ret
|
||||
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_extend_attn_backend)
|
||||
ForwardContext(attn_backend=self.draft_extend_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.draft_extend_attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
|
||||
@@ -800,7 +800,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
forward_context(
|
||||
ForwardContext(
|
||||
attn_backend=self.draft_attn_backend.attn_backends[i]
|
||||
)
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
canary_index_ctx,
|
||||
):
|
||||
@@ -903,7 +904,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
forward_context(
|
||||
ForwardContext(
|
||||
attn_backend=self.draft_attn_backend.attn_backends[i]
|
||||
)
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
canary_index_ctx,
|
||||
):
|
||||
|
||||
@@ -357,7 +357,10 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
saved_backend_pool = self.draft_attn_backend.token_to_kv_pool
|
||||
self.draft_attn_backend.token_to_kv_pool = target_pool
|
||||
try:
|
||||
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
@@ -565,7 +565,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
|
||||
with (
|
||||
self._target_kv_pool_view(forward_batch),
|
||||
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
):
|
||||
seed_output = self.draft_model_runner.forward(forward_batch).logits_output
|
||||
|
||||
@@ -608,7 +611,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
|
||||
|
||||
with (
|
||||
self._target_kv_pool_view(forward_batch),
|
||||
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self.draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
),
|
||||
):
|
||||
logits_output = self.draft_model_runner.forward(
|
||||
forward_batch
|
||||
|
||||
@@ -451,7 +451,9 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
return self._run_step_body(forward_batch, num_tokens, bs)
|
||||
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
|
||||
self.deepep_adapter.capture(is_extend_in_batch=True)
|
||||
shape_key = self._make_graph_key(bs)
|
||||
@@ -1026,7 +1028,9 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
forward_batch = r.get_forward_batch(bs)
|
||||
forward_batch = r._postprocess_forward_batch(forward_batch, bs)
|
||||
attn_backend = self.draft_extend_attn_backend_list[r.step]
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
|
||||
):
|
||||
attn_backend.init_forward_metadata_out_graph(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
@@ -1037,7 +1041,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
outs = []
|
||||
for i, (r, forward_batch, num_tokens, attn_backend) in enumerate(items):
|
||||
attn_backend.init_forward_metadata_in_graph(forward_batch)
|
||||
with forward_context(ForwardContext(attn_backend=attn_backend)):
|
||||
with forward_context(
|
||||
ForwardContext(attn_backend=attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
):
|
||||
ret = r._run_step_body(forward_batch, num_tokens, bs)
|
||||
outs.append(ret)
|
||||
if i < n - 1:
|
||||
|
||||
@@ -814,7 +814,10 @@ class UnoWorkerV2(BaseSpecWorker):
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
attn_context = (
|
||||
forward_context(ForwardContext(attn_backend=self._uno_draft_attn_backend))
|
||||
forward_context(
|
||||
ForwardContext(attn_backend=self._uno_draft_attn_backend),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
if self.tree_mode
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user