[Perf] dsv41: Integrate DeepGEMM MegaGate routing (#43262)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c82f7141ad
commit
e5dc0c363a
@@ -47,7 +47,7 @@ struct MegaMoEPreDispatchParams {
|
||||
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
|
||||
// parameter rather than a runtime check because this kernel is issue-bound: the
|
||||
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF>
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF, bool kCopyTopk>
|
||||
__global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
@@ -135,7 +135,7 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
}
|
||||
|
||||
// Copy this token's topk row (no alignment assumptions; top_k is small).
|
||||
if (tid < params.top_k) {
|
||||
if (kCopyTopk && tid < params.top_k) {
|
||||
const uint32_t off = token_id * params.top_k + tid;
|
||||
params.buf_topk_idx[off] = params.topk_idx[off];
|
||||
params.buf_topk_weights[off] = params.topk_weights[off];
|
||||
@@ -158,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
// ---- Host wrapper
|
||||
// ------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF>
|
||||
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF, bool kCopyTopk>
|
||||
struct MegaMoEPreDispatchKernel {
|
||||
static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size");
|
||||
static constexpr auto kernel_one_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF, kCopyTopk>;
|
||||
static constexpr auto kernel_multi_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF, kCopyTopk>;
|
||||
|
||||
static void
|
||||
run(const tvm::ffi::TensorView x,
|
||||
@@ -190,14 +190,16 @@ struct MegaMoEPreDispatchKernel {
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(x);
|
||||
TensorMatcher({M, K}) // topk_idx
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(topk_idx);
|
||||
TensorMatcher({M, K}) // topk_weights
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(topk_weights);
|
||||
if constexpr (kCopyTopk) {
|
||||
TensorMatcher({M, K}) // topk_idx
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(topk_idx);
|
||||
TensorMatcher({M, K}) // topk_weights
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(topk_weights);
|
||||
}
|
||||
// DeepGEMM versions expose this fp8 dispatch buffer either as raw int8
|
||||
// storage or as torch.float8_e4m3fn; the kernel writes fp8 bytes in both.
|
||||
TensorMatcher({P, H}) // buf.x
|
||||
@@ -263,7 +265,7 @@ struct MegaMoEPreDispatchKernel {
|
||||
shared_sf_stride = sf.stride(1);
|
||||
}
|
||||
|
||||
const auto pad_slots = (padded_max - num_tokens) * top_k;
|
||||
const auto pad_slots = kCopyTopk ? (padded_max - num_tokens) * top_k : 0u;
|
||||
const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size);
|
||||
const auto num_total_blocks = num_tokens + num_pad_blocks;
|
||||
|
||||
|
||||
@@ -27,6 +27,17 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.bf16_mega_gate",
|
||||
backend=KernelBackend.DEEPGEMM,
|
||||
target="sglang.kernels.ops.moe.mega_gate:bf16_mega_gate",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.moe_align_block_size",
|
||||
|
||||
@@ -36,8 +36,12 @@ def _jit_hash_topk_module():
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool):
|
||||
args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf)
|
||||
def _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size: int, write_shared_sf: bool, copy_topk: bool = True
|
||||
):
|
||||
args = make_cpp_args(
|
||||
quant_group_size, is_arch_support_pdl(), write_shared_sf, copy_topk
|
||||
)
|
||||
return load_jit(
|
||||
make_name("mega_moe_pre_dispatch"),
|
||||
*args,
|
||||
@@ -127,12 +131,13 @@ def mega_moe_pre_dispatch(
|
||||
*,
|
||||
shared_x_sf: Optional[torch.Tensor] = None,
|
||||
shared_block_m: int = 0,
|
||||
copy_topk: bool = True,
|
||||
) -> None:
|
||||
if x.shape[0] == 0 and x.stride(0) != x.shape[1]:
|
||||
x = x.new_empty(x.shape)
|
||||
# shared_block_m must match the consuming MoE's paged scale layout
|
||||
module = _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size, shared_x_sf is not None
|
||||
quant_group_size, shared_x_sf is not None, copy_topk
|
||||
)
|
||||
module.run(
|
||||
x,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def bf16_mega_gate(
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
top_k: int,
|
||||
*,
|
||||
routed_scaling_factor: float,
|
||||
ep_rank: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
image_bias: Optional[torch.Tensor] = None,
|
||||
image_token_mask: Optional[torch.Tensor] = None,
|
||||
valid_token_mask: Optional[torch.Tensor] = None,
|
||||
hash_topk_ids: Optional[torch.Tensor] = None,
|
||||
to_physical_map: Optional[torch.Tensor] = None,
|
||||
logical_count: Optional[torch.Tensor] = None,
|
||||
out: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import deep_gemm
|
||||
|
||||
if valid_token_mask is not None:
|
||||
hidden_states = torch.where(valid_token_mask[:, None], hidden_states, 0)
|
||||
|
||||
if out is None:
|
||||
ids = torch.empty(
|
||||
(hidden_states.shape[0], top_k),
|
||||
dtype=torch.int64,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
weights = torch.empty_like(ids, dtype=torch.float32)
|
||||
else:
|
||||
ids, weights = out
|
||||
deep_gemm.bf16_mega_gate(
|
||||
hidden_states,
|
||||
weight,
|
||||
top_k,
|
||||
use_shared_as_routed=False,
|
||||
num_shared_experts=0,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
ep_rank=ep_rank,
|
||||
scoring_func="sqrtsoftplus",
|
||||
mask=valid_token_mask,
|
||||
bias=bias,
|
||||
image_bias=image_bias,
|
||||
image_token_mask=image_token_mask,
|
||||
fix_routing_mask=(
|
||||
torch.ones_like(ids[:, 0], dtype=torch.bool)
|
||||
if hash_topk_ids is not None
|
||||
else None
|
||||
),
|
||||
unmapped_topk_idx=hash_topk_ids,
|
||||
to_physical_map=to_physical_map,
|
||||
logical_count=logical_count,
|
||||
out=(ids, weights),
|
||||
)
|
||||
return weights, ids
|
||||
@@ -1432,6 +1432,7 @@ class Envs:
|
||||
# residency margin so every cluster can launch beside other streams.
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2)
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True)
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_GATE = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
# Top-k kernels
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopKConfig,
|
||||
_post_process_topk_ids,
|
||||
apply_routing_simulation,
|
||||
routing_simulation_enabled,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
|
||||
# DeepGEMM bf16_mega_gate scores one token block per launch.
|
||||
_MEGA_GATE_MAX_ROWS = 2048
|
||||
|
||||
|
||||
class MegaGateRouting(NamedTuple):
|
||||
topk_weights: Optional[torch.Tensor]
|
||||
topk_ids: Optional[torch.Tensor]
|
||||
routed_scaling_factor: float
|
||||
in_buffer: bool
|
||||
|
||||
|
||||
def mega_gate_supported(moe) -> bool:
|
||||
if (
|
||||
_device_sm // 10 != 10
|
||||
or moe.config.model_type != "deepseek_v41"
|
||||
or moe.topk.enable_waterfill
|
||||
or moe.num_fused_shared_experts
|
||||
or get_exec().deterministic.enable_deterministic_inference
|
||||
):
|
||||
return False
|
||||
weight = moe.gate.weight
|
||||
num_experts, hidden = weight.shape
|
||||
top_k = moe.config.num_experts_per_tok
|
||||
if (
|
||||
weight.dtype != torch.bfloat16
|
||||
or not weight.is_contiguous()
|
||||
or hidden % 256 != 0
|
||||
or not 0 < num_experts <= 512
|
||||
or num_experts % 4 != 0
|
||||
or not 0 < top_k <= min(32, num_experts)
|
||||
):
|
||||
return False
|
||||
bias_vl = None if moe.is_hash else moe.gate.e_score_correction_bias_vl
|
||||
for bias in (moe.gate.e_score_correction_bias, bias_vl):
|
||||
if bias is not None and (
|
||||
bias.dtype != torch.float32
|
||||
or not bias.is_contiguous()
|
||||
or bias.shape != (num_experts,)
|
||||
):
|
||||
return False
|
||||
if moe.is_hash:
|
||||
return (
|
||||
moe.topk.score_func == "sqrtsoftplus"
|
||||
and moe.topk.num_shared_topk_slots == 0
|
||||
and moe.topk.topk == top_k
|
||||
)
|
||||
config = moe.topk.topk_config
|
||||
return (
|
||||
config.scoring_func == "sqrtsoftplus"
|
||||
and config.sqrtsoftplus_log1p
|
||||
and config.renormalize
|
||||
and config.num_shared_topk_slots == 0
|
||||
and config.top_k == top_k
|
||||
and not (top_k == 1 and bias_vl is not None)
|
||||
and not config.use_grouped_topk
|
||||
and config.custom_routing_function is None
|
||||
and not config.torch_native
|
||||
)
|
||||
|
||||
|
||||
def should_use_mega_gate(moe, hidden_states: torch.Tensor) -> bool:
|
||||
forward = get_forward()
|
||||
return (
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get()
|
||||
and moe.mega_gate_supported
|
||||
and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS
|
||||
and not forward.is_extend_in_batch
|
||||
and not forward.sp_active
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and hidden_states.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _captures_routed_experts(moe) -> bool:
|
||||
return (
|
||||
not moe.is_hash
|
||||
and moe.topk.topk_config.allow_routed_experts_capture
|
||||
and get_global_experts_capturer() is not None
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def _ones(num_experts: int, device: torch.device) -> torch.Tensor:
|
||||
return torch.ones(num_experts, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
def _physical_dispatch_maps(info) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if info.ep_dispatch_algorithm == "static":
|
||||
rank_map = info.partial_logical_to_rank_dispatch_physical_map
|
||||
return rank_map.to(torch.int32)[:, None], _ones(len(rank_map), rank_map.device)
|
||||
return (
|
||||
info.partial_logical_to_all_physical_map.to(torch.int32),
|
||||
info.partial_logical_to_all_physical_map_num_valid.to(torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateRouting]:
|
||||
bound, dispatch_info = moe.topk.routing_inputs()
|
||||
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm in (
|
||||
"lp",
|
||||
"fake",
|
||||
):
|
||||
return None
|
||||
if moe.is_hash and input_ids is None:
|
||||
return None
|
||||
num_tokens = hidden_states.shape[0]
|
||||
valid = None
|
||||
if bound is not None:
|
||||
valid = (
|
||||
torch.arange(num_tokens, device=hidden_states.device, dtype=torch.int32)
|
||||
< bound.reshape(-1)[:1]
|
||||
)
|
||||
image = hash_ids = None
|
||||
if moe.is_hash:
|
||||
token_ids = input_ids[:num_tokens]
|
||||
if valid is not None:
|
||||
token_ids = torch.where(valid, token_ids, 0)
|
||||
hash_ids = moe.topk.tid2eid[token_ids].to(torch.int64)
|
||||
elif moe.gate.e_score_correction_bias_vl is not None and input_ids is not None:
|
||||
image = (input_ids[:num_tokens] == moe.config.image_token_id).contiguous()
|
||||
|
||||
routed_scaling_factor = float(moe.routed_scaling_factor)
|
||||
fused_in_topk = moe.routed_scaling_factor_fused_in_topk
|
||||
# The benchmark override writes logical ids, so with EPLB it must run before the remap.
|
||||
in_buffer = (
|
||||
buf is not None
|
||||
and not _captures_routed_experts(moe)
|
||||
and not (dispatch_info is not None and routing_simulation_enabled())
|
||||
)
|
||||
to_physical_map = logical_count = None
|
||||
if in_buffer and dispatch_info is not None:
|
||||
# NOTE(kpham-sgl): MegaMoE reads physical expert slots from this buffer, so
|
||||
# under EPLB the gate must map logical ids in-kernel to stay on this path.
|
||||
to_physical_map, logical_count = _physical_dispatch_maps(dispatch_info)
|
||||
weights, ids = bf16_mega_gate(
|
||||
hidden_states,
|
||||
moe.gate.weight,
|
||||
moe.config.num_experts_per_tok,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if in_buffer or fused_in_topk else 1.0
|
||||
),
|
||||
ep_rank=get_parallel().moe_ep_rank,
|
||||
bias=moe.gate.e_score_correction_bias,
|
||||
image_bias=moe.gate.e_score_correction_bias_vl if image is not None else None,
|
||||
image_token_mask=image,
|
||||
valid_token_mask=valid,
|
||||
hash_topk_ids=hash_ids,
|
||||
to_physical_map=to_physical_map,
|
||||
logical_count=logical_count,
|
||||
out=(
|
||||
(buf.topk_idx[:num_tokens], buf.topk_weights[:num_tokens])
|
||||
if in_buffer
|
||||
else None
|
||||
),
|
||||
)
|
||||
if not moe.is_hash and routing_simulation_enabled():
|
||||
top_k = ids.shape[1]
|
||||
apply_routing_simulation(
|
||||
ids,
|
||||
weights,
|
||||
moe.gate.weight.shape[0],
|
||||
num_routed_topk=top_k,
|
||||
top_k=top_k,
|
||||
layer_id=moe.layer_id,
|
||||
num_token_non_padded=bound,
|
||||
weight_scale=(
|
||||
routed_scaling_factor if in_buffer and not fused_in_topk else 1.0
|
||||
),
|
||||
)
|
||||
if in_buffer:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=buf.topk_idx[:num_tokens]
|
||||
)
|
||||
return MegaGateRouting(None, None, 1.0, True)
|
||||
|
||||
if moe.is_hash:
|
||||
config = TopKConfig(
|
||||
top_k=moe.topk.topk,
|
||||
routed_scaling_factor=moe.topk.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=moe.topk.apply_routed_scaling_factor_on_output,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
else:
|
||||
config = moe.topk.topk_config
|
||||
ids, weights, recorder_ids = _post_process_topk_ids(
|
||||
ids,
|
||||
weights,
|
||||
config,
|
||||
hidden_states.new_empty((0, moe.gate.weight.shape[0])),
|
||||
moe.layer_id,
|
||||
num_token_non_padded=bound,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
padded_rows_masked=True,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
return MegaGateRouting(
|
||||
weights, ids, 1.0 if fused_in_topk else routed_scaling_factor, False
|
||||
)
|
||||
@@ -25,6 +25,7 @@ from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.moe.mega_gate import run_mega_gate, should_use_mega_gate
|
||||
from sglang.srt.layers.moe.mega_moe_sm90 import (
|
||||
is_sm90_fp8_mega_moe_available,
|
||||
run_sm90_mega_routed,
|
||||
@@ -250,6 +251,25 @@ def forward_mega_moe(
|
||||
return y
|
||||
|
||||
|
||||
def _mega_moe_buffer(moe: DeepseekV2MoE):
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
return _get_mega_moe_symm_buffer(
|
||||
get_parallel().moe_ep_group.device_group,
|
||||
num_experts=moe.experts.num_experts,
|
||||
num_max_tokens_per_rank=(
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
|
||||
),
|
||||
num_topk=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
|
||||
hidden=moe.config.hidden_size,
|
||||
intermediate_hidden=moe.config.moe_intermediate_size,
|
||||
num_shared_experts=(
|
||||
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
|
||||
),
|
||||
mma_type=_mega_moe_mma_type(moe.experts),
|
||||
)
|
||||
|
||||
|
||||
def _run_mega_routed(
|
||||
moe: DeepseekV2MoE,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -259,7 +279,24 @@ def _run_mega_routed(
|
||||
) -> torch.Tensor:
|
||||
hidden_size = moe.config.hidden_size
|
||||
|
||||
if num_tokens > 0:
|
||||
routing = None
|
||||
if should_use_mega_gate(moe, hidden_states):
|
||||
import deep_gemm
|
||||
|
||||
# Only the fp8xfp4 pre-dispatch can skip its top-k copy (kCopyTopk).
|
||||
buf = (
|
||||
_mega_moe_buffer(moe)
|
||||
if _mega_moe_mma_type(moe.experts) == "fp8xfp4"
|
||||
else None
|
||||
)
|
||||
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
|
||||
routing = run_mega_gate(moe, hidden_states, input_ids_global, buf)
|
||||
|
||||
if routing is not None:
|
||||
topk_ids = routing.topk_ids
|
||||
topk_weights = routing.topk_weights
|
||||
routed_scaling_factor = routing.routed_scaling_factor
|
||||
elif num_tokens > 0:
|
||||
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
|
||||
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
|
||||
if (
|
||||
@@ -282,6 +319,12 @@ def _run_mega_routed(
|
||||
else:
|
||||
topk_ids = None
|
||||
topk_weights = None
|
||||
if routing is None:
|
||||
routed_scaling_factor = (
|
||||
1.0
|
||||
if moe.routed_scaling_factor_fused_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
)
|
||||
|
||||
return run_mega_routed_experts(
|
||||
moe.experts,
|
||||
@@ -293,16 +336,13 @@ def _run_mega_routed(
|
||||
top_k=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
|
||||
num_tokens=num_tokens,
|
||||
activation_clamp=moe.experts.moe_runner_config.swiglu_limit,
|
||||
routed_scaling_factor=(
|
||||
1.0
|
||||
if moe.routed_scaling_factor_fused_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
),
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
num_shared_experts=(
|
||||
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
|
||||
),
|
||||
shared_l1_weights=moe.mega_shared_l1_weights,
|
||||
shared_l2_weights=moe.mega_shared_l2_weights,
|
||||
routing_in_buffer=routing is not None and routing.in_buffer,
|
||||
)
|
||||
|
||||
|
||||
@@ -321,6 +361,7 @@ def run_mega_routed_experts(
|
||||
num_shared_experts: int = 0,
|
||||
shared_l1_weights=None,
|
||||
shared_l2_weights=None,
|
||||
routing_in_buffer: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Rows are this rank's tokens; the returned rows are fully combined.
|
||||
import deep_gemm
|
||||
@@ -351,7 +392,10 @@ def run_mega_routed_experts(
|
||||
mma_type=mma_type,
|
||||
)
|
||||
|
||||
if num_tokens > 0:
|
||||
if routing_in_buffer:
|
||||
assert num_tokens > 0 and _device_sm != 90 and mma_type == "fp8xfp4"
|
||||
topk_ids_in = topk_weights_in = None
|
||||
elif num_tokens > 0:
|
||||
topk_ids_in = topk_ids.to(torch.int32)
|
||||
topk_weights_in = topk_weights.to(torch.float32)
|
||||
else:
|
||||
@@ -372,6 +416,7 @@ def run_mega_routed_experts(
|
||||
)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
assert not routing_in_buffer
|
||||
topk_weights_in = topk_weights_in * routed_scaling_factor
|
||||
|
||||
mega_kwargs = {"recipe": (1, 1, 32)}
|
||||
@@ -429,8 +474,8 @@ def run_mega_routed_experts(
|
||||
)
|
||||
mega_moe_pre_dispatch(
|
||||
hidden_states,
|
||||
topk_ids_in,
|
||||
topk_weights_in,
|
||||
buf.topk_idx if routing_in_buffer else topk_ids_in,
|
||||
buf.topk_weights if routing_in_buffer else topk_weights_in,
|
||||
buf.x,
|
||||
buf.x_sf,
|
||||
buf.topk_idx,
|
||||
@@ -440,6 +485,7 @@ def run_mega_routed_experts(
|
||||
buf.shared_l1_acts_sf if shared_l1_weights is not None else None
|
||||
),
|
||||
shared_block_m=shared_block_m,
|
||||
copy_topk=not routing_in_buffer,
|
||||
)
|
||||
|
||||
# Allocate at least one row so y has a non-null CUDA data_ptr;
|
||||
|
||||
@@ -467,9 +467,11 @@ def _simulate_balanced_routing_kernel(
|
||||
stride_ik,
|
||||
stride_wm,
|
||||
stride_wk,
|
||||
num_non_padded_ptr,
|
||||
K: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
RANDOM: tl.constexpr,
|
||||
HAS_NON_PADDED: tl.constexpr = False,
|
||||
):
|
||||
"""One program per token: overwrite its top-k row with a balanced expert
|
||||
assignment and uniform ``1/k`` weights, in a single launch — so the
|
||||
@@ -505,12 +507,13 @@ def _simulate_balanced_routing_kernel(
|
||||
gid = (deal % num_ranks) * experts_per_rank + (
|
||||
deal // num_ranks
|
||||
) % experts_per_rank
|
||||
w = tl.full((BLOCK_K,), inv_k, tl.float32)
|
||||
if HAS_NON_PADDED:
|
||||
is_pad = t >= tl.load(num_non_padded_ptr)
|
||||
gid = tl.where(is_pad, -1, gid)
|
||||
w = tl.where(is_pad, 0.0, w)
|
||||
tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask)
|
||||
tl.store(
|
||||
topk_weights_ptr + t * stride_wm + j * stride_wk,
|
||||
tl.full((BLOCK_K,), inv_k, tl.float32),
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(topk_weights_ptr + t * stride_wm + j * stride_wk, w, mask=mask)
|
||||
|
||||
|
||||
# Per-launch seed for the uniform (RANDOM=True) path: varies across eager calls so
|
||||
@@ -529,6 +532,8 @@ def _simulate_balanced_routing(
|
||||
token_shard_rank: int = 0,
|
||||
num_token_shards: int = 1,
|
||||
seed: Optional[int] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
weight_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Benchmark-only fused override (in place): replace ``topk_ids`` with a
|
||||
balanced expert assignment and ``topk_weights`` with ``1/k`` using a single
|
||||
@@ -589,7 +594,7 @@ def _simulate_balanced_routing(
|
||||
num_ranks,
|
||||
experts_per_rank,
|
||||
deal_offset,
|
||||
1.0 / k,
|
||||
weight_scale / k,
|
||||
seed,
|
||||
token_shard_rank,
|
||||
num_token_shards,
|
||||
@@ -597,9 +602,11 @@ def _simulate_balanced_routing(
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
num_token_non_padded if num_token_non_padded is not None else topk_ids,
|
||||
K=k,
|
||||
BLOCK_K=triton.next_power_of_2(k),
|
||||
RANDOM=random,
|
||||
HAS_NON_PADDED=num_token_non_padded is not None,
|
||||
)
|
||||
|
||||
|
||||
@@ -2401,6 +2408,8 @@ def apply_routing_simulation(
|
||||
num_routed_topk: int,
|
||||
top_k: int,
|
||||
layer_id: Optional[int],
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
weight_scale: float = 1.0,
|
||||
) -> None:
|
||||
simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
# Benchmark-only: override gating with a balanced expert assignment (so
|
||||
@@ -2441,6 +2450,8 @@ def apply_routing_simulation(
|
||||
layer_id=layer_id,
|
||||
token_shard_rank=token_shard_rank,
|
||||
num_token_shards=num_token_shards,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
weight_scale=weight_scale,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1869,6 +1869,12 @@ class DeepseekV2MoE(nn.Module):
|
||||
def _routed_mxfp8_prequant_static_enabled(self) -> bool:
|
||||
return self._compute_routed_mxfp8_prequant_enabled()[0]
|
||||
|
||||
@cached_property
|
||||
def mega_gate_supported(self) -> bool:
|
||||
from sglang.srt.layers.moe.mega_gate import mega_gate_supported
|
||||
|
||||
return mega_gate_supported(self)
|
||||
|
||||
def _compute_routed_mxfp8_prequant_enabled(self) -> Tuple[bool, str]:
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
|
||||
|
||||
|
||||
Reference in New Issue
Block a user