[Refactor] moe: Derive routing inputs from the layer and active forward (#44994)

Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
Cheng Wan
2026-10-20 16:26:15 -07:00
committed by GitHub
co-authored by Aditya Suresh
parent 533020b500
commit 299a75ac5d
83 changed files with 3915 additions and 641 deletions
@@ -327,7 +327,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.impl("ggml_moe_a8_vec", torch::kCUDA, &ggml_moe_a8_vec);
m.def("ggml_moe_get_block_size(int type) -> int");
m.impl("ggml_moe_get_block_size", torch::kCUDA, &ggml_moe_get_block_size);
m.impl("ggml_moe_get_block_size", &ggml_moe_get_block_size);
/*
* From csrc/mamba
@@ -32,6 +32,8 @@ static __device__ __forceinline__ void moe_q(
const int nrows_y,
const int nrows_dst,
const int top_k) {
if (blockIdx.y * mmq_x >= num_tokens_post_padded[0]) return;
const int blocks_per_row_x = ncols_x / qk;
const int blocks_per_col_y = nrows_y / QK8_1;
const int blocks_per_warp = WARP_SIZE_GGUF / qi;
@@ -49,8 +51,22 @@ static __device__ __forceinline__ void moe_q(
}
const int exp_idx = expert_ids[blockIdx.y];
if (exp_idx > 255 || exp_idx < 0) return;
if (blockIdx.y * mmq_x > num_tokens_post_padded[0]) return;
if (exp_idx < 0) {
#pragma unroll
for (int j = 0; j < mmq_x; j += nwarps) {
const int col_dst = token_offs[j / nwarps];
if (col_dst >= ncols_dst) continue;
#pragma unroll
for (int i = 0; i < mmq_y; i += WARP_SIZE_GGUF) {
const auto row_dst = row_dst_0 + threadIdx.x + i;
if (row_dst < nrows_dst) {
dst[col_dst * nrows_dst + row_dst] = scalar_t(0.0f);
}
}
}
return;
}
if (exp_idx > 255) return;
const block_q_t* x = (const block_q_t*)((char*)vx + exp_idx * exp_stride);
const block_q8_1* y = (const block_q8_1*)(vy);
@@ -20,6 +20,12 @@ static __global__ void moe_vec_q(
if (row >= nrows) {
return;
}
if (expert < 0) {
if (threadIdx.x == 0) {
dst[blockIdx.z * nrows + row] = scalar_t(0.0f);
}
return;
}
const int blocks_per_row = ncols / qk;
const int blocks_per_warp = vdr * WARP_SIZE / qi;
@@ -383,14 +383,17 @@ __global__ void Marlin(
int num_tokens_past_padded = num_tokens_past_padded_ptr[0];
int parallel = num_tokens_past_padded / moe_block_size;
int num_valid_blocks = parallel;
if constexpr (kIsEP) {
for (int i = 0; i < parallel; i++) {
if (expert_ids_ptr[i] == -1) num_valid_blocks--;
}
// Every warp counts the same inactive blocks without a serial per-thread scan.
int num_invalid_blocks = 0;
for (int i = threadIdx.x % 32; i < parallel; i += 32) {
num_invalid_blocks += expert_ids_ptr[i] == -1;
}
int num_invalid_blocks = parallel - num_valid_blocks;
parallel = num_valid_blocks;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
num_invalid_blocks += __shfl_xor_sync(0xffffffff, num_invalid_blocks, offset);
}
parallel -= num_invalid_blocks;
if (parallel == 0) return;
int k_tiles = prob_k / 16 / thread_k_blocks;
int n_tiles = prob_n / 16 / thread_n_blocks;
@@ -518,23 +521,18 @@ __global__ void Marlin(
if (par_id >= parallel) return;
old_expert_id = expert_id;
if constexpr (kIsEP) {
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
skip_count--;
if (num_invalid_blocks > 0) {
int skip_count = block_id == -1 ? par_id : 0;
block_id++;
for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) {
expert_id = expert_ids_ptr[i];
if (expert_id != -1) {
if (skip_count == 0) {
block_id = i;
break;
};
}
} else {
block_id = par_id;
expert_id = expert_ids_ptr[block_id];
skip_count--;
};
}
} else {
block_id = par_id;
@@ -46,8 +46,8 @@ __global__ void moe_permute_prepare_kernel(
bool is_ep) {
int tid = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
int32_t neg_count = 0;
if (is_ep) neg_count = lower_bound(sorted_topk_ids, numel, 0);
// Padding and nonlocal routes are both inactive, independently of EP.
int32_t neg_count = lower_bound(sorted_topk_ids, numel, 0);
for (int e = tid; e <= num_experts; e += stride) {
int32_t offset;
+10
View File
@@ -60,6 +60,16 @@ register_kernel(
)
register_kernel(
KernelSpec(
op="moe.shuffle_weighted_sum",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.shuffle_weighted_sum:shuffle_weighted_sum",
capabilities=CUDA_ONLY,
)
)
def moe_align_block_size(
topk_ids: torch.Tensor,
num_experts: int,
@@ -136,7 +136,6 @@ def fused_moe_kernel_gptq_awq(
use_int4_w4a16: tl.constexpr,
use_int8_w8a16: tl.constexpr,
even_Ks: tl.constexpr,
filter_expert: tl.constexpr,
):
"""
Implements the fused computation for a Mixture of Experts (MOE) using
@@ -190,10 +189,8 @@ def fused_moe_kernel_gptq_awq(
token_mask = offs_token < num_valid_tokens
off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64)
if filter_expert and off_experts == -1:
# -----------------------------------------------------------
# Write back zeros to the output when the expert is not
# in the current expert parallel rank.
if off_experts == -1:
# Padding and nonlocal routes both use -1, including without EP.
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -377,12 +374,10 @@ def fused_moe_kernel(
per_channel_quant: tl.constexpr,
even_Ks: tl.constexpr,
c_sorted: tl.constexpr,
filter_expert: tl.constexpr,
swap_ab: tl.constexpr,
FUSE_ADD_TO_OUTPUT: tl.constexpr,
MASK_OUTPUT: tl.constexpr,
FUSE_SUM_ALL_REDUCE: tl.constexpr,
LORA_PRESERVE_BASE: tl.constexpr,
ROUTER_TOPK: tl.constexpr,
FUSE_SWIGLU: tl.constexpr = False,
USE_GDC: tl.constexpr = False,
@@ -450,16 +445,10 @@ def fused_moe_kernel(
off_experts_i32 = tl.load(expert_ids_ptr + pid_m)
off_experts = off_experts_i32.to(tl.int64)
if filter_expert and off_experts == -1:
if FUSE_SWIGLU:
# C is the half-width post-activation buffer here. Rows owned by a
# filtered expert are never read (the down-GEMM CTA for this block
# early-exits before loading A), and an N-wide zero store would run
# past the row end into a neighboring token's data.
return
if not FUSE_ADD_TO_OUTPUT and not (FUSE_SUM_ALL_REDUCE and LORA_PRESERVE_BASE):
# Write zeros only when this kernel owns the full output; the experimental LoRA
# add path (LORA_PRESERVE_BASE) keeps the base output from the prior MoE kernel.
if off_experts == -1:
# The down GEMM never reads inactive sorted or half-width intermediates.
# Add/reduce outputs already hold their initial value and must stay intact.
if not (c_sorted or FUSE_SWIGLU or FUSE_ADD_TO_OUTPUT or FUSE_SUM_ALL_REDUCE):
write_zeros_to_output(
c_ptr,
stride_cm,
@@ -822,6 +811,7 @@ def invoke_fused_moe_kernel(
lora_preserve_base: bool = False,
fuse_swiglu: bool = False,
) -> None:
"""Inactive expert IDs are always skipped, including with filter_expert=False."""
assert topk_weights.stride(1) == 1
assert sorted_token_ids.stride(0) == 1
@@ -974,7 +964,6 @@ def invoke_fused_moe_kernel(
use_int4_w4a16=use_int4_w4a16,
use_int8_w8a16=use_int8_w8a16,
even_Ks=even_Ks,
filter_expert=filter_expert,
**config,
)
@@ -1046,11 +1035,9 @@ def invoke_fused_moe_kernel(
per_channel_quant=per_channel_quant,
even_Ks=even_Ks,
c_sorted=c_sorted,
filter_expert=filter_expert,
swap_ab=swap_ab,
FUSE_ADD_TO_OUTPUT=fuse_add_to_output,
MASK_OUTPUT=mask_output,
LORA_PRESERVE_BASE=lora_preserve_base,
FUSE_SUM_ALL_REDUCE=fuse_sum_all_reduce,
ROUTER_TOPK=router_topk,
FUSE_SWIGLU=fuse_swiglu,
@@ -16,7 +16,7 @@ def moe_fused_mul_sum_kernel(
num_tokens,
stride_m,
has_expert_map: tl.constexpr,
is_ep: tl.constexpr,
has_topk_ids: tl.constexpr,
top_k: tl.constexpr,
size: tl.constexpr,
routed_scaling_factor: tl.constexpr,
@@ -43,16 +43,23 @@ def moe_fused_mul_sum_kernel(
if routed_scaling_factor != 1.0:
b_val = b_val * routed_scaling_factor
if has_expert_map:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = tl.load(expert_map_ptr + id_val) >= 0
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=-1)
mapped_id = tl.load(
expert_map_ptr + tl.maximum(id_val, 0),
mask=m_mask & (id_val >= 0),
other=-1,
)
expert_mask = mapped_id >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
other=0.0,
).to(tl.float32)
elif is_ep:
elif has_topk_ids:
id_val = tl.load(top_ids_ptr + offs_m * top_k + n, mask=m_mask, other=0)
expert_mask = id_val >= 0
b_val = tl.where(expert_mask, b_val, 0.0)
a_vec = tl.load(
a_base + n * size,
mask=mask & expert_mask[:, None],
@@ -164,8 +171,8 @@ def moe_fused_mul_sum(
Shape: (num_tokens, top_k).
outputs: Optional pre-allocated output tensor.
Shape: (num_tokens, hidden_size).
topk_ids: Optional indices of the top-k experts. Used when
`expert_map` is provided. Shape: (num_tokens, top_k).
topk_ids: Optional indices of the top-k experts. Negative ids always
skip the corresponding input load. Shape: (num_tokens, top_k).
expert_map: Optional mapping for Expert Parallelism. A value < 0
indicates an invalid token/expert pair that will be skipped.
@@ -180,6 +187,7 @@ def moe_fused_mul_sum(
assert inputs.dtype in (torch.float32, torch.float16, torch.bfloat16)
assert topk_weights.dtype in (torch.float32, torch.float16, torch.bfloat16)
# Keep is_ep as a compatible host argument; route validity follows ids.
num_tokens, top_k, size = inputs.shape
output_shape = (num_tokens, size)
if outputs is None:
@@ -205,7 +213,7 @@ def moe_fused_mul_sum(
num_tokens,
top_k * size,
expert_map is not None,
is_ep,
topk_ids is not None,
top_k,
size,
1.0 if routed_scaling_factor is None else routed_scaling_factor,
@@ -68,13 +68,17 @@ def moe_wna16_marlin_gemm(
use_fp32_reduce: bool = False,
is_zp_float: bool = False,
) -> torch.Tensor:
"""Project routes, preserving inactive rows in a caller-provided output.
Inactive rows are zero when this function allocates the output.
"""
device = a.device
# Allocate output if not provided
if c_or_none is not None:
c = c_or_none
else:
c = torch.empty((size_m * top_k, size_n), dtype=a.dtype, device=device)
c = torch.zeros((size_m * top_k, size_n), dtype=a.dtype, device=device)
# Early return for zero-size M
if size_m == 0:
@@ -0,0 +1,74 @@
"""Reduce routed rows while skipping inactive permutation entries."""
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["rows", "weight_stride0", "weight_stride1"])
def _shuffle_weighted_sum_kernel(
inputs,
output,
permutation,
weights,
width: tl.constexpr,
rows,
topk: tl.constexpr,
input_stride: tl.constexpr,
output_stride: tl.constexpr,
weight_stride0,
weight_stride1,
BLOCK: tl.constexpr,
):
token = tl.program_id(0)
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
total = tl.full((BLOCK,), 0, tl.float32)
for slot in range(topk):
row = tl.load(permutation + token * topk + slot).to(tl.int64)
active = (row >= 0) & (row < rows)
values = tl.load(
inputs + row * input_stride + columns,
mask=active & (columns < width),
other=0,
).to(tl.float32)
factor = tl.load(
weights + token * weight_stride0 + slot * weight_stride1,
mask=active,
other=0,
).to(tl.float32)
total += values * factor
tl.store(
output + token.to(tl.int64) * output_stride + columns,
total,
mask=columns < width,
)
def shuffle_weighted_sum(
inputs: torch.Tensor,
output: torch.Tensor,
permutation: torch.Tensor,
weights: torch.Tensor,
) -> None:
"""Sum mapped expert outputs; inactive maps read neither data nor weights."""
assert inputs.ndim == output.ndim == weights.ndim == 2
assert inputs.shape[1] == output.shape[1]
assert inputs.stride(1) == output.stride(1) == 1
assert weights.shape[0] == output.shape[0]
assert permutation.is_contiguous() and permutation.numel() == weights.numel()
if output.numel() == 0:
return
_shuffle_weighted_sum_kernel[(output.shape[0], triton.cdiv(output.shape[1], 256))](
inputs,
output,
permutation,
weights,
output.shape[1],
inputs.shape[0],
weights.shape[1],
inputs.stride(0),
output.stride(0),
weights.stride(0),
weights.stride(1),
BLOCK=256,
)
@@ -125,6 +125,10 @@ class _StageExecutor:
# handling DP attention
forward_batch: ForwardBatch = inputs["forward_batch"]
self._child_ctx = replace(
child_ctx if child_ctx is not None else get_forward_context(),
moe_num_token_non_padded=forward_batch.moe_num_token_non_padded,
)
self._global_dp_buffer_len = forward_batch.global_dp_buffer_len
self._local_dp_buffer_len = forward_batch.tbo_padded_len
self._global_num_tokens = forward_batch.global_num_tokens_cpu
@@ -39,13 +39,16 @@ class ExpertLocationDispatchInfo:
rank_invariant: bool = False
@classmethod
def init_new(cls, layer_id: int):
def init_new(cls, layer_id: int, *, is_draft: bool = False):
if is_draft:
return None
ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm
expert_location_metadata = get_global_expert_location_metadata()
assert expert_location_metadata is not None
if ep_dispatch_algorithm is None:
return None
expert_location_metadata = get_global_expert_location_metadata()
assert expert_location_metadata is not None, (
"Expert dispatch requires placement metadata"
)
return cls(
ep_dispatch_algorithm=ep_dispatch_algorithm,
@@ -1,12 +1,12 @@
from typing import TYPE_CHECKING, Optional
import torch
from sgl_kernel_npu.norm.l1_norm import l1_norm
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import topk_ids_logical_to_physical
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
_zero_topk_weights_padded_region,
capture_routed_experts_if_allowed,
select_experts,
)
@@ -59,6 +59,8 @@ def fused_topk_npu(
)
if renormalize:
from sgl_kernel_npu.norm.l1_norm import l1_norm
topk_weights = l1_norm(
topk_weights
if topk_config.num_shared_topk_slots == 0
@@ -109,9 +111,44 @@ def fused_topk_npu(
expert_location_dispatch_info=expert_location_dispatch_info,
)
if expert_location_dispatch_info is not None:
topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info)
get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids)
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
return _finalize_topk_npu(
topk_ids=topk_ids,
topk_weights=topk_weights,
router_logits=router_logits,
topk_config=topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
layer_id=layer_id,
)
def _finalize_topk_npu(
*,
topk_ids,
topk_weights,
router_logits,
topk_config,
num_token_non_padded,
expert_location_dispatch_info,
layer_id,
):
num_shared = topk_config.num_shared_topk_slots
routed_ids = topk_ids[:, :-num_shared] if num_shared else topk_ids
capture_routed_experts_if_allowed(topk_config, layer_id, routed_ids)
routed_ids = topk_ids_logical_to_physical(routed_ids, expert_location_dispatch_info)
topk_ids = (
torch.cat([routed_ids, topk_ids[:, -num_shared:]], dim=-1)
if num_shared
else routed_ids
)
# NPU dispatch retains valid IDs; duplicating a padded row's routes onto
# expert zero can exceed per-expert capacity in fused dispatch backends.
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
recorder_ids = routed_ids
if num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_ids = routed_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
get_global_expert_distribution_recorder().on_select_experts(topk_ids=recorder_ids)
return StandardTopKOutput(topk_weights, topk_ids, router_logits)
@@ -1209,10 +1209,6 @@ class AiterAttnBackend(AttentionBackend):
bs, forward_batch.seq_lens, forward_batch.req_pool_indices, max_kv_len
)
# Build cu_seqlens_q from this batch's extend lengths. The standard
# prefill metadata path leaves self.qo_indptr unset (qo_indptr=None in
# ForwardMetadata), so relying on it corrupts multi-sequence batches
# (only bs=1 happens to work).
cu_seqlens_q = torch.zeros(bs + 1, dtype=torch.int32, device=q.device)
cu_seqlens_q[1:] = torch.cumsum(
forward_batch.extend_seq_lens.to(torch.int32), dim=0
@@ -2598,7 +2594,7 @@ class AiterAttnBackend(AttentionBackend):
self.forward_metadata = ForwardMetadata(
self.indices_updater_prefill.kv_indptr,
self.indices_updater_prefill.kv_indices,
None,
self.indices_updater_prefill.qo_indptr[: bs + 1],
None,
max(forward_batch.extend_seq_lens_cpu),
forward_batch.seq_lens_cpu.max().item(),
@@ -4423,7 +4419,7 @@ class AiterAttnBackend(AttentionBackend):
self.k_scale.reshape(1), # Q is cast at unit scale.
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
@@ -4499,7 +4495,7 @@ class AiterAttnBackend(AttentionBackend):
fp8_q_descale.reshape(1),
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
cu_k,
self.forward_metadata.max_q_len,
int(self.forward_metadata.max_kv_len),
@@ -4538,8 +4534,8 @@ class AiterAttnBackend(AttentionBackend):
fp8_q_descale.reshape(1),
k_descale.reshape(1),
v_descale.reshape(1),
self.qo_indptr[:bs0],
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
self.forward_metadata.max_q_len,
self.forward_metadata.max_q_len,
softmax_scale=layer.scaling,
@@ -4628,7 +4624,7 @@ class AiterAttnBackend(AttentionBackend):
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_cache,
v_cache,
self.qo_indptr[:bs0],
self.forward_metadata.qo_indptr[:bs0],
kv_indptr_arg,
page_table,
self.forward_metadata.max_q_len,
@@ -94,13 +94,13 @@ def forward_extend_vectorized_5d(
kv_indices_lin = torch.arange(
total_tokens, dtype=torch.int32, device=k_lin.device
)
kv_indptr_lin = backend.qo_indptr[:bs0]
kv_indptr_lin = backend.forward_metadata.qo_indptr[:bs0]
max_q = int(backend.forward_metadata.max_q_len)
o = mha_batch_prefill_func(
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_lin,
v_lin,
backend.qo_indptr[:bs0],
backend.forward_metadata.qo_indptr[:bs0],
kv_indptr_lin,
kv_indices_lin,
max_q,
@@ -187,7 +187,7 @@ def forward_extend_vectorized_5d(
q_local.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
k_lin,
v_lin,
backend.qo_indptr[:bs0],
backend.forward_metadata.qo_indptr[:bs0],
kv_indptr_lin,
kv_indices_lin,
max_q,
+5 -4
View File
@@ -10,7 +10,6 @@ from sglang.srt.utils import is_cuda
_is_cuda = is_cuda()
if _is_cuda:
from sgl_kernel import (
apply_shuffle_mul_sum,
es_fp8_blockwise_scaled_grouped_mm,
es_sm100_mxfp8_blockscaled_grouped_mm,
es_sm100_mxfp8_blockscaled_grouped_quant,
@@ -23,6 +22,7 @@ if _is_cuda:
from sglang.kernels.ops.moe.shuffle_rows_with_scales import (
shuffle_rows_with_scales,
)
from sglang.kernels.ops.moe.shuffle_weighted_sum import shuffle_weighted_sum
def cutlass_fused_experts_fp8(
@@ -142,8 +142,9 @@ def cutlass_fused_experts_fp8(
topk = topk_ids.size(1)
device = a.device
a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device)
# prepare_moe_input writes only live routes; gathers still span the tail.
a_map = torch.zeros((topk_ids.numel()), dtype=torch.int32, device=device)
c_map = torch.full((topk_ids.numel(),), -1, dtype=torch.int32, device=device)
if use_mxfp8:
assert es_up and es_down, "MXFP8 requires expert-specialization for both GEMMs"
@@ -345,5 +346,5 @@ def cutlass_fused_experts_fp8(
if output is None:
output = torch.empty((m, k), device=device, dtype=out_dtype)
apply_shuffle_mul_sum(c2, output, c_map, topk_weights.to(out_dtype))
shuffle_weighted_sum(c2, output, c_map, topk_weights.to(out_dtype))
return output
@@ -5,7 +5,6 @@ from typing import Optional
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_cuda, is_cuda_alike
_is_cuda = is_cuda()
@@ -128,8 +127,8 @@ def cutlass_w4a8_moe(
assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1"
device = a.device
if get_parallel().moe_ep_size > 1:
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
# Both graph padding and nonlocal routes use -1, including without EP.
topk_ids = torch.where(topk_ids == -1, num_local_experts, topk_ids)
src2dst = cutlass_w4_run_moe_ep_preproess(
topk_ids,
@@ -144,7 +143,11 @@ def cutlass_w4a8_moe(
# TODO: fuse per_tensor_absmax_fp8 and pre_reorder_for_cutlass_moe
if a1_scale is None:
a1_scale = torch.zeros(1, dtype=torch.float32, device=device)
per_tensor_absmax_fp8(a, a1_scale)
# Inactive rows must not change the quantization of live expert inputs.
inactive_rows = (topk_ids == num_local_experts).all(dim=-1, keepdim=True)
scale_input = a.masked_fill(inactive_rows, 0.0)
per_tensor_absmax_fp8(scale_input, a1_scale)
a1_scale.clamp_(min=1e-10 / torch.finfo(torch.float8_e4m3fn).max)
pre_reorder_for_cutlass_moe(
a,
@@ -305,15 +305,14 @@ def fused_marlin_moe(
device=hidden_states.device,
dtype=hidden_states.dtype,
)
# Marlin skips masked expert rows, so their shared cache must start at zero.
intermediate_cache13 = torch.zeros(
(M * topk_ids.shape[1] * max(gemm1_n, K),),
intermediate_cache13 = torch.empty(
(M * topk * max(gemm1_n, K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * gemm1_n]
intermediate_cache1 = intermediate_cache1.view(-1, gemm1_n)
intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K]
intermediate_cache3 = intermediate_cache13[: M * topk * K]
intermediate_cache3 = intermediate_cache3.view(-1, K)
use_atomic_add = (
@@ -381,9 +380,9 @@ def fused_marlin_moe(
else:
raise ValueError(f"Unsupported activation: {activation=}, with {is_gated=}")
if expert_map is not None:
intermediate_cache3.zero_()
# Gate-up is dead after activation, so its storage can hold the down output.
# Skipped down routes must start at zero before the top-k reduction.
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
intermediate_cache3,
@@ -74,6 +74,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_global_dwdp_manager,
get_parallel,
get_server_args,
@@ -367,7 +368,8 @@ class FusedMoE(torch.nn.Module):
self.params_dtype = params_dtype
self.layer_name = prefix
self.layer_id = layer_id
self.uses_expert_location = uses_expert_location
self.is_draft = get_flags().moe.in_speculative_scope
self.uses_expert_location = uses_expert_location and not self.is_draft
self.top_k = top_k
self.hidden_size = hidden_size
self.num_experts = num_experts
@@ -1559,6 +1561,14 @@ class FusedMoE(torch.nn.Module):
return forward_fuseep(self, hidden_states, topk_output)
if is_in_tc_piecewise_cuda_graph():
if (
TopKOutputChecker.format_is_bypassed(topk_output)
and topk_output.expert_location_dispatch_info is not None
):
# The opaque logits-only op cannot carry placement metadata.
return self.forward_impl(
hidden_states, topk_output, pre_quant_input=pre_quant_input
)
if TopKOutputChecker.format_is_standard(topk_output):
return moe_forward_piecewise_cuda_graph_impl(
hidden_states,
@@ -1872,7 +1882,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
) -> torch.Tensor:
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=TopKConfig(
top_k=top_k,
topk_group=topk_group,
@@ -1880,6 +1890,7 @@ def fused_moe_bypassed_piecewise_cuda_graph_impl(
correction_bias=correction_bias,
renormalize=renormalize,
allow_routed_experts_capture=allow_routed_experts_capture,
layer_id=layer_id,
),
)
forward_context = get_tc_piecewise_forward_context()
+38 -20
View File
@@ -21,11 +21,10 @@ from sglang.srt.layers.moe.topk import (
_mask_topk_ids_padded_region,
_zero_topk_weights_padded_region,
remap_topk_for_per_rank_shared_slots,
resolve_routing_inputs,
)
from sglang.srt.layers.moe.utils import (
has_per_rank_fused_shared_slots,
)
from sglang.srt.runtime_context import get_model
from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots
from sglang.srt.runtime_context import get_flags, get_model
from sglang.srt.utils import is_hip, is_npu, is_xpu
logger = logging.getLogger(__name__)
@@ -46,9 +45,13 @@ class HashTopK(nn.Module):
routed_scaling_factor=1.5,
apply_routed_scaling_factor_on_output=False,
layer_id: Optional[int] = None,
is_draft: Optional[bool] = None,
):
super().__init__()
self.layer_id = layer_id
self.is_draft = (
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
)
self.shared_experts = get_model().shared_experts
self.enable_waterfill = self.shared_experts.use_waterfill
@@ -75,6 +78,9 @@ class HashTopK(nn.Module):
"with fused shared experts; pass --disable-shared-experts-fusion."
)
def routing_inputs(self):
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
@property
def num_shared_topk_slots(self) -> int:
return self.shared_experts.num_shared_topk_slots
@@ -100,7 +106,8 @@ class HashTopK(nn.Module):
self, device: torch.device, *, layer_id: Optional[int] = None
):
topk = self.topk - self.num_shared_topk_slots
if layer_id is not None:
if not self.is_draft:
layer_id = self.layer_id
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
lplb_solver = get_global_lplb_solver(layer_id)
@@ -226,6 +233,12 @@ class HashTopK(nn.Module):
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
):
num_token_non_padded, expert_location_dispatch_info = resolve_routing_inputs(
layer_id=self.layer_id,
is_draft=self.is_draft,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
)
assert input_ids.shape[0] == hidden_states.shape[0] == router_logits.shape[0], (
f"{input_ids.shape=} {hidden_states.shape=} {router_logits.shape=}"
)
@@ -252,6 +265,9 @@ class HashTopK(nn.Module):
topk_weights = topk_weights * self.routed_scaling_factor
num_shared_topk_slots = self.num_shared_topk_slots
routed_ids = (
topk_ids[:, :-num_shared_topk_slots] if num_shared_topk_slots else topk_ids
)
log2phy_prob = None
if (
expert_location_dispatch_info is not None
@@ -263,18 +279,13 @@ class HashTopK(nn.Module):
lplb_solver = get_global_lplb_solver(self.layer_id)
if lplb_solver is not None:
log2phy_prob = lplb_solver.solve(topk_ids)
log2phy_prob = lplb_solver.solve(routed_ids)
recorder_topk_ids = None
routed_cols = topk_ids_logical_to_physical(
routed_ids, expert_location_dispatch_info, log2phy_prob
)
recorder_topk_ids = routed_cols if num_shared_topk_slots else None
if has_per_rank_fused_shared_slots(num_shared_topk_slots):
# The trailing columns are placeholders; the remap appends the real ones.
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots],
expert_location_dispatch_info,
log2phy_prob,
)
recorder_topk_ids = routed_cols
num_physical_routed_experts = (
expert_location_dispatch_info.num_physical_experts
if expert_location_dispatch_info is not None
@@ -292,11 +303,13 @@ class HashTopK(nn.Module):
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
),
)
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
elif num_shared_topk_slots:
topk_ids = torch.cat(
[routed_cols, topk_ids[:, -num_shared_topk_slots:]], dim=-1
)
if is_hip():
else:
topk_ids = routed_cols
if _is_hip or _is_npu:
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
else:
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
@@ -304,6 +317,11 @@ class HashTopK(nn.Module):
_mask_topk_ids_padded_region(recorder_topk_ids, num_token_non_padded)
if recorder_topk_ids is None:
recorder_topk_ids = topk_ids
if _is_npu and num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_topk_ids = recorder_topk_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_topk_ids
)
@@ -311,7 +329,7 @@ class HashTopK(nn.Module):
topk_weights=topk_weights, topk_ids=topk_ids, router_logits=router_logits
)
topk_output = self._apply_waterfill(topk_output, hidden_states.shape[0])
if is_hip():
if _is_hip or _is_npu:
_zero_topk_weights_padded_region(
topk_output.topk_weights, num_token_non_padded
)
-15
View File
@@ -23,7 +23,6 @@ import torch
from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
from sglang.srt.layers.moe.mega_moe_sm90 import (
@@ -263,16 +262,6 @@ def _run_mega_routed(
if num_tokens > 0:
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
num_token_non_padded = (
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
if moe.experts.uses_expert_location
else None
)
if (
not moe.is_hash
and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None
@@ -281,15 +270,11 @@ def _run_mega_routed(
moe,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
)
else:
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
topk_ids = topk_output.topk_ids
@@ -14,7 +14,6 @@ import torch
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
@@ -307,25 +306,16 @@ def _run_mega_routed(
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.num_token_non_padded
if forward_batch is not None
else None
),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
if moe.experts.uses_expert_location
else None
),
**topk_kwargs,
)
x_in = hidden_states
topk_ids = topk_output.topk_ids.to(torch.int32)
topk_weights = topk_output.topk_weights.to(torch.float32)
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
num_token_non_padded, _ = moe.topk.routing_inputs()
if num_token_non_padded is not None:
_fill_padded_rows(
topk_ids,
forward_batch.num_token_non_padded,
num_token_non_padded,
experts,
)
else:
+158 -20
View File
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
from sglang.srt.configs.shared_experts import SharedExpertsConfig
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_forward,
get_lora,
get_model,
@@ -110,6 +111,10 @@ from sglang.srt.layers.moe.utils import (
RoutingMethodType,
has_per_rank_fused_shared_slots,
)
from sglang.srt.model_executor.forward_context import (
get_forward_context,
has_forward_context,
)
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import (
cpu_has_amx_support,
@@ -279,6 +284,8 @@ class TopKConfig:
# Draft-side MoE blocks set this False so they never write the target's
# process-global routed-experts capture buffer.
allow_routed_experts_capture: bool = True
# Lazy materialization still needs the producer's solver and capture slot.
layer_id: Optional[int] = None
_correction_bias_dtype_cache: Optional[torch.Tensor] = field(
default=None, init=False, repr=False, compare=False
)
@@ -393,10 +400,10 @@ class TritonKernelTopKOutput(NamedTuple):
class BypassedTopKOutput(NamedTuple):
"""Bypassed top-k output format."""
"""Lazy routing that can be materialized or consumed by an in-kernel router."""
hidden_states: torch.Tensor
router_logits: torch.Tensor
raw_router_logits: torch.Tensor
topk_config: TopKConfig
num_token_non_padded: Optional[torch.Tensor] = None
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None
@@ -405,14 +412,23 @@ class BypassedTopKOutput(NamedTuple):
def format(self) -> TopKOutputFormat:
return TopKOutputFormat.BYPASSED
@property
def router_logits(self) -> torch.Tensor:
if self.expert_location_dispatch_info is not None:
raise NotImplementedError(
"Expert placement remapping requires materialized TopK routing; "
"this MoE kernel routes directly from logits."
)
return self.raw_router_logits
def to_standard(self, layer_id: Optional[int] = None) -> StandardTopKOutput:
"""Materialize routing tensors. Used by MoE kernels that need explicit
topk_ids / topk_weights rather than doing routing internally."""
return select_experts(
hidden_states=self.hidden_states,
router_logits=self.router_logits,
router_logits=self.raw_router_logits,
topk_config=self.topk_config,
layer_id=layer_id,
layer_id=self.topk_config.layer_id if layer_id is None else layer_id,
num_token_non_padded=self.num_token_non_padded,
expert_location_dispatch_info=self.expert_location_dispatch_info,
)
@@ -630,6 +646,24 @@ def _can_bypass_trtllm_routing(config: TopKConfig) -> bool:
# -------------------------------- TopK ---------------------------------------
def resolve_routing_inputs(
*,
layer_id: int,
is_draft: bool,
num_token_non_padded: Optional[torch.Tensor] = None,
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
):
if num_token_non_padded is None and has_forward_context():
get_bound = get_forward_context().moe_num_token_non_padded
if get_bound is not None:
num_token_non_padded = get_bound()
if is_draft:
expert_location_dispatch_info = None
elif expert_location_dispatch_info is None:
expert_location_dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id)
return num_token_non_padded, expert_location_dispatch_info
class TopK(BaseFusedOp):
"""
Parameters:
@@ -649,6 +683,7 @@ class TopK(BaseFusedOp):
top_k: int,
*,
layer_id: int,
is_draft: Optional[bool] = None,
use_grouped_topk: bool = False,
topk_group: Optional[int] = None,
num_expert_group: Optional[int] = None,
@@ -674,6 +709,9 @@ class TopK(BaseFusedOp):
assert num_expert_group is not None and topk_group is not None
self.layer_id = layer_id
self.is_draft = (
get_flags().moe.in_speculative_scope if is_draft is None else is_draft
)
self._runner_backend_bound = False
shared_experts = get_model().shared_experts
@@ -686,6 +724,7 @@ class TopK(BaseFusedOp):
self.is_fp4_experts = is_fp4_experts
self.topk_config = TopKConfig(
top_k=top_k + num_shared_slots,
layer_id=layer_id,
use_grouped_topk=use_grouped_topk,
renormalize=renormalize,
topk_group=topk_group,
@@ -703,7 +742,21 @@ class TopK(BaseFusedOp):
scoring_func=scoring_func,
sqrtsoftplus_log1p=sqrtsoftplus_log1p,
fused_gate_packed_ids=fused_gate_packed_ids,
allow_routed_experts_capture=allow_routed_experts_capture,
allow_routed_experts_capture=allow_routed_experts_capture
and not self.is_draft,
)
def routing_inputs(self):
return resolve_routing_inputs(layer_id=self.layer_id, is_draft=self.is_draft)
def _resolve_routing_inputs(
self, num_token_non_padded, expert_location_dispatch_info
):
return resolve_routing_inputs(
layer_id=self.layer_id,
is_draft=self.is_draft,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
)
def bind_runner_backend(
@@ -778,6 +831,11 @@ class TopK(BaseFusedOp):
dynamic_expert_bias: Optional[torch.Tensor] = None,
router_logits_partials: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if router_logits_partials is not None:
rocm_router_reduce_partials(router_logits_partials, router_logits)
self.topk_config.torch_native = True
@@ -805,6 +863,12 @@ class TopK(BaseFusedOp):
"""router_logits_partials (ROCm decode router): fp32 split-K partials whose
fixed-order sum is the logits; router_logits is then a buffer the fused gate
fills, and any other reader first reduces the partials into it."""
explicit_dispatch_info = expert_location_dispatch_info is not None
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if dynamic_expert_bias is not None:
output_format = TopKOutputFormat.STANDARD
elif self.topk_config.output_format is not None:
@@ -814,6 +878,17 @@ class TopK(BaseFusedOp):
"TopK requires the owning experts' runner backend or an explicit output format."
)
requires_remapping = expert_location_dispatch_info is not None and (
explicit_dispatch_info
or is_expert_remapping_enabled()
or expert_location_dispatch_info.ep_dispatch_algorithm in ("fake", "lp")
)
if output_format == TopKOutputFormat.TRITON_KERNEL and requires_remapping:
raise NotImplementedError(
"Expert placement remapping requires a MoE runner with standard TopK routing; "
f"got {output_format.name}."
)
if (
router_logits_partials is not None
and output_format != TopKOutputFormat.STANDARD
@@ -833,6 +908,7 @@ class TopK(BaseFusedOp):
router_logits,
self.topk_config.top_k,
sm_first=not self.topk_config.renormalize,
n_rows=num_token_non_padded,
)
return TritonKernelTopKOutput(
a_ragged_metadata,
@@ -844,10 +920,12 @@ class TopK(BaseFusedOp):
elif output_format == TopKOutputFormat.BYPASSED:
return BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
raw_router_logits=router_logits,
topk_config=self.topk_config,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
expert_location_dispatch_info=(
expert_location_dispatch_info if requires_remapping else None
),
)
else:
self.topk_config.torch_native = False
@@ -875,6 +953,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
topk_output = select_experts(
hidden_states=hidden_states,
layer_id=self.layer_id,
@@ -895,6 +978,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
if dynamic_expert_bias is not None:
self.topk_config.torch_native = False
return select_experts(
@@ -923,12 +1011,13 @@ class TopK(BaseFusedOp):
) -> TopKOutput:
"""Return an empty topk output for a rank with zero tokens this forward.
When ``layer_id`` is provided and the active dispatch algorithm is LP,
also calls ``LPLBSolver.solve(empty)`` so that this rank participates
in the EP all-reduce. Without this, an empty rank would skip the
collective and deadlock under DP-attention.
Target layers call their owning ``LPLBSolver.solve(empty)`` so this
rank participates in the EP all-reduce even without tokens. Draft
layers skip the target solver. The optional ``layer_id`` is retained
for caller compatibility; the router's owning layer determines the solver.
"""
if layer_id is not None:
if not self.is_draft:
layer_id = self.layer_id
# Skip the full ExpertLocationDispatchInfo allocation — we only
# need the per-layer solver to participate in the EP all-reduce.
from sglang.srt.eplb.lplb_solver import get_global_lplb_solver
@@ -972,6 +1061,11 @@ class TopK(BaseFusedOp):
expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None,
dynamic_expert_bias: Optional[torch.Tensor] = None,
) -> TopKOutput:
num_token_non_padded, expert_location_dispatch_info = (
self._resolve_routing_inputs(
num_token_non_padded, expert_location_dispatch_info
)
)
self.topk_config.torch_native = True
# [NOTE] XPU device support for topk kernels
# - support 'topk_softmax' and 'topk_sigmoid'
@@ -1816,6 +1910,12 @@ def _mask_topk_ids_padded_region(
) -> None:
if num_token_non_padded is None:
return
if not topk_ids.is_cuda:
if _is_npu:
return
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
topk_ids.masked_fill_((rows >= num_token_non_padded)[:, None], fill_value)
return
# TODO: let the kernel support other dtypes
if _is_cuda and topk_ids.dtype == torch.int32 and fill_value == -1:
mask_topk_ids(topk_ids, num_token_non_padded)
@@ -1834,6 +1934,10 @@ def _zero_topk_weights_padded_region(
):
if num_token_non_padded is None:
return
if not topk_weights.is_cuda:
rows = torch.arange(topk_weights.shape[0], device=topk_weights.device)
topk_weights.masked_fill_((rows >= num_token_non_padded)[:, None], 0.0)
return
if _can_fuse_padded_region(topk_weights):
_fill_padded_rows(topk_weights, num_token_non_padded, 0.0)
return
@@ -2420,12 +2524,27 @@ def _post_process_topk_ids(
lplb_solver = get_global_lplb_solver(layer_id)
if lplb_solver is not None:
log2phy_prob = lplb_solver.solve(topk_ids)
lp_routed_ids = (
topk_ids[:, :-num_shared_topk_slots]
if router_wrote_shared_cols
else topk_ids
)
log2phy_prob = lplb_solver.solve(lp_routed_ids)
if log2phy_prob is not None:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
)
if router_wrote_shared_cols:
shared_cols = topk_ids[:, -num_shared_topk_slots:]
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots],
expert_location_dispatch_info,
log2phy_prob,
)
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
recorder_topk_ids = topk_ids[:, :-num_shared_topk_slots]
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info, log2phy_prob
)
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
elif router_wrote_shared_cols:
# Shared IDs are outside EPLB's routed-expert table for both global
@@ -2506,6 +2625,21 @@ def _post_process_topk_ids(
# That final pass re-zeros after any shared-expert append/remap, so a
# second zeroing here would be redundant (zeroing is idempotent).
else:
if num_shared_topk_slots > 0:
shared_cols = topk_ids[:, -num_shared_topk_slots:]
routed_cols = topk_ids_logical_to_physical(
topk_ids[:, :-num_shared_topk_slots], expert_location_dispatch_info
)
_mask_topk_ids_padded_region(routed_cols, num_token_non_padded)
topk_ids = torch.cat([routed_cols, shared_cols], dim=-1)
recorder_topk_ids = routed_cols
else:
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info
)
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded)
if recorder_topk_ids is None and not recorder_was_fused:
recorder_topk_ids = (
topk_ids[:, :-num_shared_topk_slots] if _gate_appended else topk_ids
@@ -2598,6 +2732,14 @@ def _post_process_topk_ids(
# supplied 1/ep_size factor the shared contribution is summed ep_size times.
topk_weights[:, -num_shared_topk_slots:] *= fused_shared_experts_scaling_factor
if _is_npu:
_zero_topk_weights_padded_region(topk_weights, num_token_non_padded)
if recorder_topk_ids is not None and num_token_non_padded is not None:
rows = torch.arange(topk_ids.shape[0], device=topk_ids.device)
recorder_topk_ids = recorder_topk_ids.masked_fill(
(rows >= num_token_non_padded)[:, None], -1
)
if _is_hip and not _skip_hip_pad_mask and not _fold_pad_weights_into_append:
# Shared-expert append/remap can introduce non-zero weights after the
# initial HIP padding mask above. Ensure padded tokens leave this helper
@@ -2750,10 +2892,6 @@ def select_experts(
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
)
elif torch_native and custom_routing_function is None:
assert num_token_non_padded is None, (
"num_token_non_padded is not yet supported in fused_topk_native"
)
assert expert_location_dispatch_info is None
fold_scale_after_routing = True
topk_weights, topk_ids = fused_topk_native(
hidden_states=hidden_states,
@@ -387,6 +387,9 @@ class MuseSpark1xFp4Experts(nn.Module):
from sglang.kernels.ops.activation.activation import silu_and_mul
from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm
from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
from sglang.srt.layers.moe.token_dispatcher.standard import (
_map_topk_ids_to_local_experts,
)
from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace
hidden_states = hidden_states.contiguous()
@@ -398,7 +401,7 @@ class MuseSpark1xFp4Experts(nn.Module):
local_topk_ids = topk_ids.to(torch.int32)
if expert_map is not None:
local_topk_ids = expert_map[local_topk_ids]
local_topk_ids = _map_topk_ids_to_local_experts(local_topk_ids, expert_map)
local_topk_ids = local_topk_ids.contiguous()
topk_weights = topk_scores.to(torch.float32).contiguous()
@@ -430,13 +433,13 @@ class MuseSpark1xFp4Experts(nn.Module):
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache13 = torch.empty(
(num_tokens * topk * max(gemm1_n, size_k),),
cache13 = torch.zeros(
(num_tokens * topk * (gemm1_n + size_k),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
cache1 = cache13[: num_tokens * topk * gemm1_n].view(-1, gemm1_n)
cache3 = cache13[: num_tokens * topk * size_k].view(-1, size_k)
cache3 = cache13[num_tokens * topk * gemm1_n :].view(-1, size_k)
cache1 = moe_wna16_marlin_gemm(
hidden_states,
@@ -525,6 +528,7 @@ class MuseSpark1xFp4Experts(nn.Module):
expert_outputs, local_topk_ids = self._marlin_expert_outputs(
hidden_states, topk_scores, topk_ids
)
topk_scores = topk_scores.masked_fill(local_topk_ids < 0, 0.0)
if self._post_expert_norm is not post_expert_norm:
self.set_post_expert_norm(post_expert_norm)
if self._post_expert_norm_weight is not None:
+39 -15
View File
@@ -18,6 +18,9 @@ from sglang.srt.hardware_backend.npu.quantization.moe_methods import (
from sglang.srt.hardware_backend.npu.utils import npu_format_cast
from sglang.srt.layers.linear import LinearBase
from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.moe_align_block_size import (
moe_align_block_size,
)
from sglang.srt.layers.moe.utils import MoeRunnerBackend, get_moe_runner_backend
from sglang.srt.layers.quantization.base_config import (
FusedMoEMethodBase,
@@ -41,24 +44,22 @@ _is_musa = is_musa()
_is_npu = is_npu()
if _is_cuda:
from sgl_kernel import moe_align_block_size, moe_sum
from sgl_kernel import moe_sum
from sgl_kernel.quantization import (
ggml_dequantize,
ggml_moe_a8,
ggml_moe_a8_vec,
ggml_moe_get_block_size,
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
elif _is_musa:
from sgl_kernel import gelu_and_mul, moe_align_block_size, moe_sum, silu_and_mul
from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul
from sgl_kernel.quantization import (
ggml_dequantize,
ggml_moe_a8,
ggml_moe_a8_vec,
ggml_moe_get_block_size,
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
@@ -180,6 +181,16 @@ MMVQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES | IMATRIX_QUANT_TYPES
MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
def ggml_moe_get_block_size(qweight_type: int) -> int:
# MOE_X in the CUDA/MUSA GGUF MMQ kernels is 4. The released wheel's
# Tensor-free metadata operator has no catch-all dispatch registration.
return (
(8 if _is_hip else 4)
if qweight_type in MMQ_QUANT_TYPES and qweight_type != WeightType.Q8_1
else 0
)
def dequantize_gguf_weight(
qweight: torch.Tensor, qweight_type: int, dtype: torch.dtype
) -> torch.Tensor:
@@ -239,6 +250,8 @@ def fused_moe_gguf(
return gelu_and_mul(x)
raise ValueError(f"Unsupported activation: {activation}")
inactive = topk_ids < 0
x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0)
out_hidden_states = torch.empty_like(x)
# unless we decent expert reuse we are better off running moe_vec kernel
if (
@@ -254,6 +267,11 @@ def fused_moe_gguf(
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids, BLOCK_SIZE, E
)
# Released MMQ kernels inspect one slack block before their bound check.
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
)
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
out = ggml_moe_a8(
x,
w1,
@@ -265,6 +283,7 @@ def fused_moe_gguf(
top_k,
num_tokens,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = ggml_moe_a8(
out,
@@ -277,8 +296,10 @@ def fused_moe_gguf(
1,
num_tokens * top_k,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
topk_weights.view(num_tokens, top_k, 1)
valid_weights.view(num_tokens, top_k, 1)
)
# TODO(FlamingoPg): maybe we can use moe_sum_reduce here?
moe_sum(out, out_hidden_states)
@@ -287,14 +308,20 @@ def fused_moe_gguf(
E, N, _ = w1.shape
top_k = topk_ids.shape[1]
out = ggml_moe_a8_vec(x, w1, topk_ids, top_k, qweight_type, N, num_tokens)
# Older vector kernels form weight addresses before checking -1.
# They have no per-expert capacity buffer; discard their dummy slots.
safe_ids = topk_ids.clamp_min(0)
out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = ggml_moe_a8_vec(
out, w2, topk_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
topk_weights.view(num_tokens, top_k, 1)
valid_weights.view(num_tokens, top_k, 1)
)
moe_sum(out, out_hidden_states)
else:
@@ -305,8 +332,10 @@ def fused_moe_gguf(
)
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
inp = x[tok].reshape((1,) + x.shape[1:])
current_hidden_state = None
current_hidden_state = torch.zeros_like(inp)
for ww, ii in zip(w, idx):
if ii < 0:
continue
expert_up = w1[ii]
out = fused_mul_mat_gguf(inp, expert_up, qweight_type)
@@ -316,10 +345,7 @@ def fused_moe_gguf(
current_state = fused_mul_mat_gguf(
out, expert_down, qweight_type2
).mul_(ww)
if current_hidden_state is None:
current_hidden_state = current_state
else:
current_hidden_state.add_(current_state)
current_hidden_state.add_(current_state)
out_hidden_states[tok] = current_hidden_state
return out_hidden_states
@@ -565,8 +591,6 @@ class GGUFMoEMethod(FusedMoEMethodBase):
layer: torch.nn.Module,
dispatch_output: StandardDispatchOutput,
) -> CombineInput:
assert self.fused_experts is None
from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput
assert self.moe_runner_config.activation == "silu", (
@@ -71,8 +71,8 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
assert runner_config.activation == "silu", "Only SiLU activation is supported."
assert torch.cuda.get_device_capability(hidden_states.device)[0] >= 9, (
@@ -119,9 +119,13 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
)
# Stage 1: Gate/Up (Marlin)
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -165,11 +169,6 @@ class MarlinLoraRunnerCore(DispatchMoeRunnerCore):
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: Down (Marlin)
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
if quant_info.expert_map is not None:
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -1,7 +1,7 @@
"""No-sort B200 decode kernels for Inkling routed MoE LoRA.
The contract is BF16 CUDA, M<=32, top-k 6, rank 32, up to four slots, TP-local
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)``.
intermediate 384 or 768, and non-EP expert IDs in ``[0, E)`` or the inactive sentinel ``-1``.
"""
from __future__ import annotations
@@ -47,7 +47,8 @@ def _direct_gate_expand_kernel(
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_r = tl.arange(0, RANK)
@@ -110,7 +111,8 @@ def _direct_down_shrink_kernel(
expert = tl.load(topk_ids_ptr + route).to(tl.int64)
token = route // ROUTER_TOPK
slot = tl.load(token_lora_mapping_ptr + token).to(tl.int64)
active = (slot >= 0) & (slot < NUM_SLOTS)
active = (slot >= 0) & (slot < NUM_SLOTS) & (expert >= 0)
expert = tl.maximum(expert, 0)
offs_m = tl.arange(0, BLOCK_M)
offs_r = tl.arange(0, RANK)
offs_k = tl.arange(0, BLOCK_K)
@@ -121,7 +123,7 @@ def _direct_down_shrink_kernel(
k_mask = k < INTERMEDIATE_SIZE
x = tl.load(
activation_ptr + route * stride_xq + k[None, :] * stride_xk,
mask=(offs_m[:, None] == 0) & k_mask[None, :],
mask=active & (offs_m[:, None] == 0) & k_mask[None, :],
other=0.0,
)
a = tl.load(
@@ -232,8 +232,8 @@ def fused_experts_experimental_sgl_marlin_lora(
hidden_states = dispatch_output.hidden_states
topk_output = dispatch_output.topk_output
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights.masked_fill(topk_ids < 0, 0.0)
if (
topk_ids.ndim != 2
or topk_weights.shape != topk_ids.shape
@@ -345,7 +345,7 @@ def fused_experts_experimental_sgl_marlin_lora(
!= lora_info.gate_up_lora_b_weights.shape[-1]
)
# Allocate and prewarm on main; side streams launch kernels only.
gate_up_delta = hidden_states.new_empty((M, topk, 2 * N))
gate_up_delta = hidden_states.new_zeros((M, topk, 2 * N))
if not direct_decode:
merged_experts_fused_moe_lora_add(
output=gate_up_delta,
@@ -572,9 +572,13 @@ def fused_experts_experimental_sgl_marlin_lora(
)
# Stage 1: gate_up (marlin) — concurrent with the side-stream delta above.
intermediate_cache1 = torch.empty(
(M * topk, 2 * N), device=hidden_states.device, dtype=hidden_states.dtype
projection_cache = torch.zeros(
(M * topk * (2 * N + K),),
device=hidden_states.device,
dtype=hidden_states.dtype,
)
intermediate_cache1 = projection_cache[: M * topk * 2 * N].view(-1, 2 * N)
intermediate_cache3 = projection_cache[M * topk * 2 * N :].view(-1, K)
intermediate_cache1 = moe_wna16_marlin_gemm(
hidden_states,
intermediate_cache1,
@@ -657,8 +661,7 @@ def fused_experts_experimental_sgl_marlin_lora(
local_expert_offset=0,
local_num_experts=E,
zero_intermediate=(
ep_active
and lora_info.experts_shared_outer_loras
lora_info.experts_shared_outer_loras
and not factored_shared_outer
),
)
@@ -679,11 +682,6 @@ def fused_experts_experimental_sgl_marlin_lora(
silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2)
# Stage 3: down (marlin).
intermediate_cache3 = torch.empty(
(M * topk, K), device=hidden_states.device, dtype=hidden_states.dtype
)
if ep_active:
intermediate_cache3.zero_()
intermediate_cache3 = moe_wna16_marlin_gemm(
intermediate_cache2,
@@ -764,7 +762,7 @@ def fused_experts_experimental_sgl_marlin_lora(
routing_cache=routing_cache,
local_expert_offset=0,
local_num_experts=E,
zero_intermediate=(ep_active and lora_info.experts_shared_outer_loras),
zero_intermediate=lora_info.experts_shared_outer_loras,
)
# Never alias hidden_states: the shared sink still reads it. The fused B200
@@ -866,9 +864,7 @@ def fused_experts_experimental_sgl_marlin_lora(
down_intermediate if down_shrink_done is not None else None
),
zero_intermediate=(
ep_active
and lora_info.experts_shared_outer_loras
and down_shrink_done is None
lora_info.experts_shared_outer_loras and down_shrink_done is None
),
)
@@ -835,7 +835,9 @@ class CPUGraphRunner:
spec_algorithm=self.model_runner.spec_algorithm,
spec_info=spec_info,
capture_hidden_mode=self.capture_hidden_mode,
num_token_non_padded=self.num_token_non_padded,
num_token_non_padded=(
self.num_token_non_padded if enable_num_token_non_padded() else None
),
global_forward_mode=self.capture_forward_mode,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
@@ -849,7 +851,8 @@ class CPUGraphRunner:
)
with skip_ctx:
with forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=forward_batch,
):
self.model_runner.attn_backend.init_forward_metadata_capture_cpu_graph(
bs,
@@ -991,11 +994,15 @@ class CPUGraphRunner:
with replay_context():
with skip_ctx:
prepared_forward_batch = self.prepare_replay(forward_batch, skip=skip)
output = graphs[prepared_forward_batch.batch_size](
prepared_forward_batch.input_ids,
prepared_forward_batch.positions,
prepared_forward_batch,
)
with forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=prepared_forward_batch,
):
output = graphs[prepared_forward_batch.batch_size](
prepared_forward_batch.input_ids,
prepared_forward_batch.positions,
prepared_forward_batch,
)
if forward_batch.batch_size in graphs:
return output
@@ -1143,14 +1143,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
batch.extend_input_logprob_token_ids.to(device, non_blocking=True)
)
num_tokens = len(batch.input_ids) if batch.input_ids is not None else 0
if enable_num_token_non_padded():
ret.global_num_token_non_padded = torch.tensor(
num_tokens,
dtype=torch.int32,
pin_memory=is_pin_memory_available(device),
).to(device, non_blocking=True)
ret.global_num_token_non_padded_cpu = num_tokens
ret._init_num_token_non_padded(
device=device, is_draft_worker=model_runner.is_draft_worker
)
ret.init_mlp_sync_metadata(
batch, device, is_draft_worker=model_runner.is_draft_worker
@@ -1341,6 +1336,26 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
pin_memory=is_pin_memory_available(batch.device),
).to(batch.device, non_blocking=True)
def _init_num_token_non_padded(self, *, device, is_draft_worker: bool) -> None:
if self.input_ids is not None:
num_tokens = len(self.input_ids)
elif (
is_draft_worker
and self.spec_info is not None
and self.forward_mode.is_decode()
):
assert self.spec_info.num_tokens_per_req > 0
num_tokens = self.batch_size * self.spec_info.num_tokens_per_req
else:
num_tokens = 0
if enable_num_token_non_padded():
self.global_num_token_non_padded = torch.tensor(
num_tokens,
dtype=torch.int32,
pin_memory=is_pin_memory_available(device),
).to(device, non_blocking=True)
self.global_num_token_non_padded_cpu = num_tokens
def set_local_num_token_non_padded(self, *, sharded: bool) -> None:
"""Derive the LOCAL num_token_non_padded from the invariant GLOBAL scalar.
@@ -5,6 +5,8 @@ layer reads at depth via get_forward_context(). The only mandatory field
today is attn_backend; pool refs are derived from attn_backend.*
(every backend caches them at __init__), so a published ForwardContext
is enough to resolve the active pools without a separate global.
The active batch lends its MoE bound getter so late preparation and TBO child
selection stay visible without retaining a parent tensor snapshot.
ModelRunner._forward_raw publishes a fresh ForwardContext for the
duration of each forward; callers that need a per-call override (PDmux
@@ -23,12 +25,15 @@ worker threads ever share a process, migrate to contextvars.ContextVar.
from __future__ import annotations
from contextlib import contextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Callable, Optional
if TYPE_CHECKING:
import torch
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@dataclass(frozen=True, slots=True)
@@ -38,6 +43,8 @@ class ForwardContext:
write time — use dataclasses.replace for per-call overrides."""
attn_backend: AttentionBackend
# Resolve after batch preparation; captured calls retain the static buffer.
moe_num_token_non_padded: Optional[Callable[[], Optional[torch.Tensor]]] = None
_current: Optional[ForwardContext] = None
@@ -76,7 +83,15 @@ def get_req_to_token_pool() -> ReqToTokenPool:
@contextmanager
def forward_context(ctx: ForwardContext):
def forward_context(
ctx: ForwardContext, *, forward_batch: Optional[ForwardBatch] = None
):
if forward_batch is not None:
ctx = replace(
ctx, moe_num_token_non_padded=forward_batch.moe_num_token_non_padded
)
elif ctx.moe_num_token_non_padded is None and _current is not None:
ctx = replace(ctx, moe_num_token_non_padded=_current.moe_num_token_non_padded)
prev = set_forward_context(ctx)
try:
yield
@@ -113,6 +113,7 @@ from sglang.srt.model_executor.forward_batch_info import (
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.graph_memory_usage import (
@@ -1965,9 +1966,14 @@ class ModelRunner:
if prepare is not None:
prepare(forward_batch)
if has_forward_context():
ctx_mgr = contextlib.nullcontext()
ctx_mgr = forward_context(
get_forward_context(), forward_batch=forward_batch
)
else:
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
ctx_mgr = forward_context(
ForwardContext(attn_backend=self.attn_backend),
forward_batch=forward_batch,
)
with (
get_model().override(shared_experts=self.model.shared_experts_config),
ctx_mgr,
@@ -762,7 +762,9 @@ class BaseRunner(ABC):
torch.get_device_module(mr.device).synchronize()
get_parallel().tp_group.barrier()
with forward_context(ForwardContext(attn_backend=mr.attn_backend)):
with forward_context(
ForwardContext(attn_backend=mr.attn_backend), forward_batch=forward_batch
):
with run_ctx or empty_context():
run_once()
@@ -1253,7 +1253,9 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
# All setup hooks below read get_attn_backend() (TboForwardBatchPreparer,
# DeepEP adapter, …) so they must run inside the same ForwardContext
# that wraps the warmup/capture forward.
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
self.tbo_plugin.capture_one_batch_size(forward_batch, num_tokens=num_tokens)
if forward_batch.lora_ids is not None:
@@ -1262,60 +1264,67 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
def run_once():
# Backends may gather live pool mappings into their static
# metadata here; these nodes run before any model KV reads.
attn_backend.init_forward_metadata_in_graph(forward_batch)
self._record_in_graph_metadata_prep_done()
forward_batch.dp_local_start_pos = forward_batch.dp_local_num_tokens = (
None
)
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
num_tokens,
forward_batch.dp_padding_mode.is_max_len(),
forward_batch.global_num_tokens_cpu,
)
set_is_extend_in_batch(False)
kwargs = {}
if (
self.pp_size > 1
and "pp_proxy_tensors" in inspect.signature(forward).parameters
# Debug replay retains this callback beyond the capture scope.
with forward_context(
ForwardContext(attn_backend=attn_backend),
forward_batch=forward_batch,
):
kwargs["pp_proxy_tensors"] = PPProxyTensors(
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
)
if (
(
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
if self.dllm_input_preparation is None:
kwargs["input_embeds"] = self.buffers.input_embeds[:num_tokens]
else:
state = self.buffer_registry.get_slot(
"dllm_input_preparation_state"
).buffer[:num_tokens]
kwargs["input_embeds"] = self.dllm_input_preparation(
self.model_runner.model, forward_batch.input_ids, state
)
# Backends may gather live pool mappings into their static
# metadata here; these nodes run before any model KV reads.
attn_backend.init_forward_metadata_in_graph(forward_batch)
self._record_in_graph_metadata_prep_done()
out = forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
**kwargs,
)
for capture_hook in self.model_runner.capture_tail_hooks:
capture_hook(self, out, forward_batch, num_tokens)
return out
forward_batch.dp_local_start_pos = (
forward_batch.dp_local_num_tokens
) = None
set_dp_buffer_len(
forward_batch.global_dp_buffer_len,
num_tokens,
forward_batch.dp_padding_mode.is_max_len(),
forward_batch.global_num_tokens_cpu,
)
set_is_extend_in_batch(False)
kwargs = {}
if (
self.pp_size > 1
and "pp_proxy_tensors" in inspect.signature(forward).parameters
):
kwargs["pp_proxy_tensors"] = PPProxyTensors(
{k: v.clone() for k, v in pp_proxy_tensors.tensors.items()}
)
if (
(
self.dllm_uses_input_embeds
or (
self.model_runner.spec_algorithm.is_dflash_family()
and self.model_runner.is_draft_worker
)
)
and "input_embeds" in inspect.signature(forward).parameters
and not hasattr(self.model_runner.model, "forward_embed")
):
if self.dllm_input_preparation is None:
kwargs["input_embeds"] = self.buffers.input_embeds[
:num_tokens
]
else:
state = self.buffer_registry.get_slot(
"dllm_input_preparation_state"
).buffer[:num_tokens]
kwargs["input_embeds"] = self.dllm_input_preparation(
self.model_runner.model, forward_batch.input_ids, state
)
out = forward(
forward_batch.input_ids,
forward_batch.positions,
forward_batch,
**kwargs,
)
for capture_hook in self.model_runner.capture_tail_hooks:
capture_hook(self, out, forward_batch, num_tokens)
return out
self.deepep_adapter.capture(is_extend_in_batch=False)
canary_ctx = (
@@ -46,6 +46,7 @@ from sglang.srt.model_executor.forward_batch_info import (
from sglang.srt.model_executor.forward_context import (
ForwardContext,
forward_context,
get_forward_context,
get_req_to_token_pool,
get_token_to_kv_pool,
)
@@ -259,7 +260,11 @@ class EagerRunner(BaseRunner):
ctx = device_timer_ctx(model_runner.device_timer, "decode")
with ctx, pdmux_ctx:
with (
ctx,
pdmux_ctx,
forward_context(get_forward_context(), forward_batch=forward_batch),
):
return model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
@@ -341,7 +346,10 @@ class EagerRunner(BaseRunner):
if forward_batch.forward_mode.is_target_verify()
else "extend"
)
with device_timer_ctx(model_runner.device_timer, category):
with (
device_timer_ctx(model_runner.device_timer, category),
forward_context(get_forward_context(), forward_batch=forward_batch),
):
if cp_active and (
not get_parallel().enable_cp_tp_group_sharing
or getattr(model_runner.model, "prepare_cp_inputs", None) is not None
@@ -448,7 +456,10 @@ class EagerRunner(BaseRunner):
model_runner.attn_backend.forward_metadata = None
kwargs = model_runner._pp_kwargs(pp_proxy_tensors)
with device_timer_ctx(model_runner.device_timer, "idle"):
with (
device_timer_ctx(model_runner.device_timer, "idle"),
forward_context(get_forward_context(), forward_batch=forward_batch),
):
return model_runner.model.forward(
forward_batch.input_ids,
forward_batch.positions,
@@ -819,7 +819,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
full_graph=self._is_full_backend, raw_num_tokens=raw_num_tokens
),
forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=forward_batch,
),
set_tc_piecewise_forward_context(
forward_batch,
@@ -953,7 +954,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
set_forward_batch(fb),
prefill_graph_scope(full_graph=self._is_full_backend),
forward_context(
ForwardContext(attn_backend=self.model_runner.attn_backend)
ForwardContext(attn_backend=self.model_runner.attn_backend),
forward_batch=fb,
),
set_tc_piecewise_forward_context(
fb,
@@ -1234,7 +1236,9 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
and stash the returned per-bucket metadata object; otherwise fall
back to the generic eager init that BCG/TC_PIECEWISE use today."""
attn_backend = self.model_runner.attn_backend
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
if not self.use_captured_attn_metadata:
attn_backend.init_forward_metadata(forward_batch)
return
@@ -20,7 +20,12 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
set_graph_pool_id,
)
from sglang.srt.distributed.parallel_state import graph_capture
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.forward_context import (
forward_context,
get_attn_backend,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
BreakableCUDAGraph,
BreakableCUDAGraphCapture,
@@ -120,7 +125,14 @@ class RowBucketGraphRunner:
# The eager breaks read the step's real row count off this view.
step_batch = copy.copy(forward_batch)
step_batch.global_num_token_non_padded_cpu = num_rows
with context(step_batch):
with (
(
forward_context(get_forward_context(), forward_batch=step_batch)
if has_forward_context()
else nullcontext()
),
context(step_batch),
):
if (rows, graph_key) not in self._graphs:
# Largest first, so smaller buckets reuse the pool memory the
# larger ones took; a bucket below this step's rows cannot be
@@ -168,7 +180,14 @@ class RowBucketGraphRunner:
forward_batch.num_token_non_padded, num_rows
)
capture_batch.num_token_non_padded = num_token_non_padded
with batch_scope(capture_batch, inputs):
with (
(
forward_context(get_forward_context(), forward_batch=capture_batch)
if has_forward_context()
else nullcontext()
),
batch_scope(capture_batch, inputs),
):
graph = self._record(
lambda: body(inputs, capture_batch),
rows,
-5
View File
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -492,10 +491,6 @@ class BailingMoESparseMoeBlock(nn.Module):
hidden_states,
router_logits,
dynamic_expert_bias=dynamic_expert_bias,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -21,7 +21,6 @@ from sglang.srt.configs import KimiLinearConfig
from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
@@ -832,10 +831,6 @@ class BailingMoE(nn.Module):
hidden_states,
router_logits,
dynamic_expert_bias=dynamic_expert_bias,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
final_hidden_states = self.experts(hidden_states, topk_output)
+7 -52
View File
@@ -66,7 +66,6 @@ from sglang.srt.distributed import divide
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.amx_utils import PackWeightMethod
@@ -120,10 +119,10 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
DispatchOutput,
)
from sglang.srt.layers.moe.topk import (
BypassedTopKOutput,
TopK,
TopKOutputFormat,
fused_shared_expert_weight,
is_expert_remapping_enabled,
is_routed_scaling_factor_fused_in_topk,
)
from sglang.srt.layers.moe.utils import (
@@ -1038,7 +1037,8 @@ class DeepseekV2MoE(nn.Module):
and not self._enable_a2a_moe
and not self._fuse_shared_experts_inside_sbo
and not getattr(self, "is_hash", False)
and not get_exec().moe.enable_eplb
and not is_expert_remapping_enabled()
and get_exec().moe.ep_dispatch_algorithm not in ("fake", "lp")
)
def forward(
@@ -1067,11 +1067,6 @@ class DeepseekV2MoE(nn.Module):
input_ids_global=input_ids_global,
)
num_token_non_padded = (
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
)
use_vision_topk = self.gate.e_score_correction_bias_vl is not None
if use_vision_topk and _is_hip:
use_vision_topk = _hip_moe.batch_has_images(forward_batch)
@@ -1109,7 +1104,6 @@ class DeepseekV2MoE(nn.Module):
gemm_output_zero_allocator,
input_ids,
input_ids_global=input_ids_global,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
@@ -1121,7 +1115,6 @@ class DeepseekV2MoE(nn.Module):
input_ids,
input_ids_global=input_ids_global,
skip_shared_experts=skip_shared_experts,
num_token_non_padded=num_token_non_padded,
use_vision_topk=use_vision_topk,
return_moe_output=return_moe_output,
leave_output_sum=leave_output_sum,
@@ -1137,7 +1130,6 @@ class DeepseekV2MoE(nn.Module):
gemm_output_zero_allocator: BumpAllocator = None,
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
@@ -1167,11 +1159,6 @@ class DeepseekV2MoE(nn.Module):
has_shared_output = (
hidden_states.shape[0] > 0 and self.num_fused_shared_experts == 0
)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb and not self.is_nextn
else None
)
# router_logits: (num_tokens, n_experts)
if _is_hip:
@@ -1185,11 +1172,7 @@ class DeepseekV2MoE(nn.Module):
router_logits = self.gate(hidden_states, gemm_output_zero_allocator)
router_logits_partials = None
if use_flashinfer_trtllm_bypass:
topk_output = BypassedTopKOutput(
hidden_states=hidden_states,
router_logits=router_logits,
topk_config=self.topk.topk_config,
)
topk_output = self.topk.forward_cuda(hidden_states, router_logits)
else:
topk_kwargs = (
{"input_ids": input_ids_global}
@@ -1203,14 +1186,11 @@ class DeepseekV2MoE(nn.Module):
self,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
)
else:
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
# Issued after the router so the main chain stays on the main stream at replay.
@@ -1323,7 +1303,6 @@ class DeepseekV2MoE(nn.Module):
input_ids: Optional[torch.Tensor] = None,
input_ids_global: Optional[torch.Tensor] = None,
skip_shared_experts: bool = False,
num_token_non_padded: Optional[torch.Tensor] = None,
use_vision_topk: bool = False,
return_moe_output: bool = False,
leave_output_sum: bool = False,
@@ -1332,11 +1311,6 @@ class DeepseekV2MoE(nn.Module):
self.shared_experts.gate_up_proj
):
return self.forward_cpu(hidden_states)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb and not self.is_nextn
else None
)
defer_shared = not self.experts.moe_runner_config.inplace
# PoC (SGLANG_DP_SHARED_EXPERT_LOCAL): shared expert is computed on the LOCAL
# hidden in the decoder layer (before the dp gather) and added after the
@@ -1383,21 +1357,18 @@ class DeepseekV2MoE(nn.Module):
self,
router_logits,
input_ids_global,
num_token_non_padded=num_token_non_padded,
)
else:
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=dispatch_info,
**topk_kwargs,
)
else:
pre_quant_input = None
shared_output = None
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
if self._fuse_shared_experts_inside_sbo and not skip_shared_experts:
@@ -1591,19 +1562,11 @@ class DeepseekV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
**topk_kwargs,
)
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
if sbo_overlap_dispatch_flag:
@@ -1979,19 +1942,11 @@ class DeepseekV2MoE(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
**topk_kwargs,
)
else:
state.topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
def op_dispatch_a(self, state):
@@ -28,7 +28,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
check_cuda_graph_backend,
)
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.forward_context import (
forward_context,
get_attn_backend,
get_forward_context,
has_forward_context,
)
from sglang.srt.model_executor.model_runner_components.layer_setup import (
compute_attention_and_moe_layers,
)
@@ -163,6 +168,15 @@ class DecoderReplayTail:
max_rows=max_rows,
)
@staticmethod
def _can_capture_moe_bound(forward_batch: ForwardBatch) -> bool:
# The row runner replaces only the local count with a graph-owned buffer.
bound = forward_batch.moe_num_token_non_padded()
return bound is None or (
bound is forward_batch.num_token_non_padded
and bound is not forward_batch.global_num_token_non_padded
)
def plan(
self,
tail: Optional[LateLayerTail],
@@ -199,6 +213,7 @@ class DecoderReplayTail:
or get_global_expert_distribution_recorder().recording
or not self.runner.warm
or (tail is not None and tail.cp_metadata is not None)
or not self._can_capture_moe_bound(forward_batch)
):
return False, None
if not dp:
@@ -414,7 +429,14 @@ class DecoderReplayTail:
gathered = input_ids.new_empty((sum(dp_rows), 1))
dp_gather_replicate(gathered, input_ids[:, None].clone(), tail_batch)
input_ids_global = gathered.squeeze(-1)
with self._break_context(tail_batch):
with (
(
forward_context(get_forward_context(), forward_batch=tail_batch)
if has_forward_context()
else nullcontext()
),
self._break_context(tail_batch),
):
return self._layer_stack(
state,
positions=positions,
@@ -681,6 +703,7 @@ def _tail_layers_in_prefill_graph(
if (
not get_flags().dp.capturing_prefill_graph
and runner.warm
and tail_layers._can_capture_moe_bound(forward_batch)
and runner.bucket_rows(max(dp_rows) if dp else num_rows) is not None
):
residual, pre, aux = tail_layers.run(**kwargs)
@@ -51,7 +51,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers import deep_gemm_wrapper
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.attention.dsa.dsa_indexer import Indexer
@@ -467,14 +466,10 @@ class Dots3MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(
-5
View File
@@ -26,7 +26,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -228,10 +227,6 @@ class ExaoneMoESparseMoEBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -36,7 +36,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
@@ -653,10 +652,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -30,7 +30,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
)
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import (
@@ -377,10 +376,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -34,7 +34,6 @@ from triton.language.extra import libdevice
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -918,10 +917,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+10
View File
@@ -97,6 +97,7 @@ from sglang.srt.model_loader.utils import (
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
get_stream,
)
@@ -251,6 +252,15 @@ class LongcatFlashMoE(nn.Module):
self.top_k = config.moe_topk
self.zero_expert_num = config.zero_expert_num
self.zero_expert_type = config.zero_expert_type
if (
self.zero_expert_num > 0
and get_exec().moe.ep_dispatch_algorithm is not None
):
raise ValueError(
"LongCat zero experts do not support expert placement remapping. "
"Disable EPLB, redundant experts, custom expert placement, and "
"--ep-dispatch-algorithm for this model."
)
if config.rounter_params_dtype == "float32":
self.rounter_params_dtype = torch.float32
-14
View File
@@ -29,7 +29,6 @@ from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -559,11 +558,6 @@ class MiMoV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if self._uses_expert_remap
else None
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -595,10 +589,6 @@ class MiMoV2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -628,10 +618,6 @@ class MiMoV2MoE(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -29,7 +29,6 @@ from transformers import PretrainedConfig
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
AuxHiddenStateList,
@@ -614,10 +613,6 @@ class MiniMaxM2MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(device=hidden_states.device)
-5
View File
@@ -36,7 +36,6 @@ from sglang.srt.configs.model_config import (
)
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -535,10 +534,6 @@ class MiniMaxM3MoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+3 -35
View File
@@ -29,7 +29,6 @@ from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import ReLU2
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
@@ -349,7 +348,6 @@ class NemotronHMoE(nn.Module):
def _forward_core(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
forward_batch: ForwardBatch | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if (
@@ -364,21 +362,16 @@ class NemotronHMoE(nn.Module):
and is_in_breakable_cuda_graph()
and is_sm120()
):
return self._forward_core_prefill_overlap(
hidden_states, num_token_non_padded
)
return self._forward_core_prefill_overlap(hidden_states)
if _is_cuda and (
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
):
return self._forward_core_shared_routed_overlap(
hidden_states, num_token_non_padded
)
return self._forward_core_normal(hidden_states, num_token_non_padded)
return self._forward_core_shared_routed_overlap(hidden_states)
return self._forward_core_normal(hidden_states)
def _forward_core_normal(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# router_scores: [num_tokens, num_experts]
# bf16 gemm on tensor cores with fp32 accumulation/output for sigmoid/topk.
@@ -392,12 +385,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -407,7 +394,6 @@ class NemotronHMoE(nn.Module):
def _forward_core_shared_routed_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
alt_stream = _get_or_create_alt_stream(self.device_module)
@@ -427,12 +413,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -444,7 +424,6 @@ class NemotronHMoE(nn.Module):
def _forward_core_prefill_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# A multi-stream graph per hybrid segment creates one internal stream
# per executable. Keep each graph linear and launch shared experts on
@@ -460,12 +439,6 @@ class NemotronHMoE(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
@@ -509,11 +482,6 @@ class NemotronHMoE(nn.Module):
# MoE runner / topk), so final_hidden_states is already scaled.
final_hidden_states, shared_output = self._forward_core(
hidden_states,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
forward_batch=forward_batch,
)
-19
View File
@@ -38,7 +38,6 @@ from sglang.srt.distributed import (
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStateAccumulator,
@@ -611,14 +610,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
)
if not self.is_nextn
else None
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -667,16 +658,6 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.enable_shared_expert_fusion:
topk_output = self._append_shared_to_topk_output(
+1 -28
View File
@@ -28,7 +28,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateAccumulator
from sglang.srt.layers.layer_boundary import (
append_stages,
@@ -342,20 +341,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
dispatch_info = (
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if get_exec().moe.enable_eplb
else None
)
if hidden_states.shape[0] > 0:
# router_logits: (num_tokens, n_experts)
router_logits, _ = self.gate(hidden_states)
topk_output = self.topk(
hidden_states,
router_logits,
expert_location_dispatch_info=dispatch_info,
)
topk_output = self.topk(hidden_states, router_logits)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
final_hidden_states = self.experts(hidden_states, topk_output)
@@ -370,10 +359,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
@@ -408,14 +393,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.moe_num_token_non_padded()
if forward_batch is not None
else None
),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
assert TopKOutputChecker.format_is_standard(topk_output), (
"MegaMoE pre-dispatch consumes raw topk ids/weights; "
@@ -454,10 +431,6 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -13,7 +13,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
@@ -159,10 +158,6 @@ class SDARMoeSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
-5
View File
@@ -6,7 +6,6 @@ import torch.nn.functional as F
from torch import nn
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
@@ -256,10 +255,6 @@ class Step3p5MoEMLP(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
+2 -7
View File
@@ -38,7 +38,6 @@ from transformers import PretrainedConfig
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary import (
@@ -1000,10 +999,6 @@ class XllmSparseMoeBlock(nn.Module):
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
),
)
# Apply router scaling factor after renormalization
if self.router_scaling_factor != 1.0:
@@ -1014,7 +1009,7 @@ class XllmSparseMoeBlock(nn.Module):
topk_output.topk_weights = scaled_weights
else:
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(
hidden_states=hidden_states,
@@ -1051,7 +1046,7 @@ class XllmSparseMoeBlock(nn.Module):
if hidden_states.shape[0] == 0:
shared_output = None
topk_output = self.topk.empty_topk_output(
hidden_states.device, layer_id=self.layer_id
hidden_states.device,
)
final_hidden_states = self.experts(hidden_states, topk_output)
else:
@@ -2,6 +2,7 @@ import torch
import torch.nn.functional as F
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location_dispatch import transform_select_experts_inputs
from sglang.srt.layers.moe.topk import (
_RENORMALIZE_SUM_EPSILON,
StandardTopKOutput,
@@ -12,19 +13,10 @@ from sglang.srt.layers.moe.topk import (
)
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
has_per_rank_fused_shared_slots,
)
from sglang.srt.utils import is_cuda
def _scale_fused_shared_weights(weights, num_fused_shared_experts, scaling_factor):
# Standard EP replicates the fused shared expert on every rank and all-reduces,
# so the shared columns carry a 1/ep_size factor.
if num_fused_shared_experts and scaling_factor is not None:
weights[:, -num_fused_shared_experts:] *= scaling_factor
return weights
def vision_topk(
moe,
logits,
@@ -32,25 +24,35 @@ def vision_topk(
num_token_non_padded=None,
expert_location_dispatch_info=None,
):
bound, dispatch_info = moe.topk.routing_inputs()
if num_token_non_padded is None:
num_token_non_padded = bound
if expert_location_dispatch_info is None:
expert_location_dispatch_info = dispatch_info
config = moe.topk.topk_config
logits, correction_bias = transform_select_experts_inputs(
logits, moe.gate.e_score_correction_bias, expert_location_dispatch_info
)
correction_bias_vl = moe.gate.e_score_correction_bias_vl
if (
expert_location_dispatch_info is not None
and expert_location_dispatch_info.ep_dispatch_algorithm == "fake"
):
correction_bias_vl = torch.zeros_like(correction_bias_vl)
num_shared_topk_slots = config.num_shared_topk_slots
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if num_shared_topk_slots and not use_mega_moe:
# This path bypasses _post_process_topk_ids, which appends the per-rank slots.
assert not has_per_rank_fused_shared_slots(num_shared_topk_slots), (
"VL routing does not support per-rank fused shared slots"
)
if is_cuda():
from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate
from sglang.srt.layers.moe.utils import get_moe_runner_backend
# Same admission as _fused_gate_emits_packed_ids: the shared-expert slots
# rescaled below would rewrite weights after the router.
packed_topk = None
if (
num_shared_topk_slots == 0
and expert_location_dispatch_info is None
and not use_mega_moe
and get_moe_runner_backend().is_flashinfer_mxfp4()
and config.runner_backend is not None
and config.runner_backend.is_flashinfer_mxfp4()
):
packed_topk = torch.empty(
(logits.shape[0], config.top_k), dtype=torch.int32, device=logits.device
@@ -58,11 +60,11 @@ def vision_topk(
weights, indices = moe_fused_gate(
logits,
moe.gate.e_score_correction_bias,
correction_bias,
topk=config.top_k,
scoring_func="sqrtsoftplus",
num_fused_shared_experts=num_shared_topk_slots,
bias_alt=moe.gate.e_score_correction_bias_vl,
bias_alt=correction_bias_vl,
input_ids=input_ids,
bias_alt_token_id=moe.config.image_token_id,
renormalize=config.renormalize and config.top_k > 1,
@@ -87,12 +89,12 @@ def vision_topk(
return StandardTopKOutput(weights, indices, logits)
scores = F.softplus(logits.float()).sqrt()
if input_ids is None:
bias = moe.gate.e_score_correction_bias
bias = correction_bias
else:
bias = torch.where(
(input_ids == moe.config.image_token_id)[:, None],
moe.gate.e_score_correction_bias_vl,
moe.gate.e_score_correction_bias,
correction_bias_vl,
correction_bias,
)
# The shared slots appended below use the same layout as biased_grouped_topk_gpu.
topk_routed = config.top_k - num_shared_topk_slots
@@ -136,28 +138,19 @@ def _finish_vision_topk(
):
config = moe.topk.topk_config
use_mega_moe = get_moe_a2a_backend().is_megamoe()
if use_mega_moe:
# Use the same EPLB mapping, recording and home-rank shared slots as
# ordinary top-k before handing physical expert IDs to MegaMoE.
indices, weights, recorder_ids = _post_process_topk_ids(
indices,
weights,
config,
logits,
moe.layer_id,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
padded_rows_masked=padded_rows_masked,
)
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids
)
else:
weights = _scale_fused_shared_weights(
weights,
config.num_shared_topk_slots,
config.fused_shared_experts_scaling_factor,
indices, weights, recorder_ids = _post_process_topk_ids(
indices,
weights,
config,
logits,
moe.layer_id,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=expert_location_dispatch_info,
padded_rows_masked=padded_rows_masked,
)
if recorder_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_ids
)
# Shared-slot remapping can overwrite padded weights; mask last so idle
# and partially padded DP ranks never dispatch a phantom shared expert.
@@ -511,7 +511,10 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch.positions.sub_(self.eagle_worker.speculative_num_steps - 1)
return ret
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
with forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
):
self.draft_attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
)
@@ -482,7 +482,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
return ret
with forward_context(
ForwardContext(attn_backend=self.draft_extend_attn_backend)
ForwardContext(attn_backend=self.draft_extend_attn_backend),
forward_batch=forward_batch,
):
self.draft_extend_attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
@@ -800,7 +800,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
forward_context(
ForwardContext(
attn_backend=self.draft_attn_backend.attn_backends[i]
)
),
forward_batch=forward_batch,
),
canary_index_ctx,
):
@@ -903,7 +904,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
forward_context(
ForwardContext(
attn_backend=self.draft_attn_backend.attn_backends[i]
)
),
forward_batch=forward_batch,
),
canary_index_ctx,
):
@@ -357,7 +357,10 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
saved_backend_pool = self.draft_attn_backend.token_to_kv_pool
self.draft_attn_backend.token_to_kv_pool = target_pool
try:
with forward_context(ForwardContext(attn_backend=self.draft_attn_backend)):
with forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
):
self.frozen_kv_mtp_worker._init_frozen_kv_metadata_capture_cuda_graph(
forward_batch
)
@@ -565,7 +565,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
with (
self._target_kv_pool_view(forward_batch),
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
),
):
seed_output = self.draft_model_runner.forward(forward_batch).logits_output
@@ -608,7 +611,10 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
with (
self._target_kv_pool_view(forward_batch),
forward_context(ForwardContext(attn_backend=self.draft_attn_backend)),
forward_context(
ForwardContext(attn_backend=self.draft_attn_backend),
forward_batch=forward_batch,
),
):
logits_output = self.draft_model_runner.forward(
forward_batch
@@ -451,7 +451,9 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
attn_backend.init_forward_metadata_in_graph(forward_batch)
return self._run_step_body(forward_batch, num_tokens, bs)
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
attn_backend.init_forward_metadata_out_graph(forward_batch, in_capture=True)
self.deepep_adapter.capture(is_extend_in_batch=True)
shape_key = self._make_graph_key(bs)
@@ -1026,7 +1028,9 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
forward_batch = r.get_forward_batch(bs)
forward_batch = r._postprocess_forward_batch(forward_batch, bs)
attn_backend = self.draft_extend_attn_backend_list[r.step]
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend), forward_batch=forward_batch
):
attn_backend.init_forward_metadata_out_graph(
forward_batch, in_capture=True
)
@@ -1037,7 +1041,10 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
outs = []
for i, (r, forward_batch, num_tokens, attn_backend) in enumerate(items):
attn_backend.init_forward_metadata_in_graph(forward_batch)
with forward_context(ForwardContext(attn_backend=attn_backend)):
with forward_context(
ForwardContext(attn_backend=attn_backend),
forward_batch=forward_batch,
):
ret = r._run_step_body(forward_batch, num_tokens, bs)
outs.append(ret)
if i < n - 1:
@@ -814,7 +814,10 @@ class UnoWorkerV2(BaseSpecWorker):
else contextlib.nullcontext()
)
attn_context = (
forward_context(ForwardContext(attn_backend=self._uno_draft_attn_backend))
forward_context(
ForwardContext(attn_backend=self._uno_draft_attn_backend),
forward_batch=forward_batch,
)
if self.tree_mode
else contextlib.nullcontext()
)