[Perf] dsv41: Integrate DeepGEMM MegaGate routing (#43262)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c82f7141ad
commit
e5dc0c363a
@@ -47,7 +47,7 @@ struct MegaMoEPreDispatchParams {
|
||||
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
|
||||
// parameter rather than a runtime check because this kernel is issue-bound: the
|
||||
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF>
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF, bool kCopyTopk>
|
||||
__global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
@@ -135,7 +135,7 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
}
|
||||
|
||||
// Copy this token's topk row (no alignment assumptions; top_k is small).
|
||||
if (tid < params.top_k) {
|
||||
if (kCopyTopk && tid < params.top_k) {
|
||||
const uint32_t off = token_id * params.top_k + tid;
|
||||
params.buf_topk_idx[off] = params.topk_idx[off];
|
||||
params.buf_topk_weights[off] = params.topk_weights[off];
|
||||
@@ -158,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
// ---- Host wrapper
|
||||
// ------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF>
|
||||
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF, bool kCopyTopk>
|
||||
struct MegaMoEPreDispatchKernel {
|
||||
static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size");
|
||||
static constexpr auto kernel_one_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF, kCopyTopk>;
|
||||
static constexpr auto kernel_multi_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF, kCopyTopk>;
|
||||
|
||||
static void
|
||||
run(const tvm::ffi::TensorView x,
|
||||
@@ -190,14 +190,16 @@ struct MegaMoEPreDispatchKernel {
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(x);
|
||||
TensorMatcher({M, K}) // topk_idx
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(topk_idx);
|
||||
TensorMatcher({M, K}) // topk_weights
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(topk_weights);
|
||||
if constexpr (kCopyTopk) {
|
||||
TensorMatcher({M, K}) // topk_idx
|
||||
.with_dtype<int32_t>()
|
||||
.with_device(device)
|
||||
.verify(topk_idx);
|
||||
TensorMatcher({M, K}) // topk_weights
|
||||
.with_dtype<float>()
|
||||
.with_device(device)
|
||||
.verify(topk_weights);
|
||||
}
|
||||
// DeepGEMM versions expose this fp8 dispatch buffer either as raw int8
|
||||
// storage or as torch.float8_e4m3fn; the kernel writes fp8 bytes in both.
|
||||
TensorMatcher({P, H}) // buf.x
|
||||
@@ -263,7 +265,7 @@ struct MegaMoEPreDispatchKernel {
|
||||
shared_sf_stride = sf.stride(1);
|
||||
}
|
||||
|
||||
const auto pad_slots = (padded_max - num_tokens) * top_k;
|
||||
const auto pad_slots = kCopyTopk ? (padded_max - num_tokens) * top_k : 0u;
|
||||
const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size);
|
||||
const auto num_total_blocks = num_tokens + num_pad_blocks;
|
||||
|
||||
|
||||
@@ -27,6 +27,17 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.bf16_mega_gate",
|
||||
backend=KernelBackend.DEEPGEMM,
|
||||
target="sglang.kernels.ops.moe.mega_gate:bf16_mega_gate",
|
||||
capabilities=frozenset(
|
||||
{CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))}
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="moe.moe_align_block_size",
|
||||
|
||||
@@ -36,8 +36,12 @@ def _jit_hash_topk_module():
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool):
|
||||
args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf)
|
||||
def _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size: int, write_shared_sf: bool, copy_topk: bool = True
|
||||
):
|
||||
args = make_cpp_args(
|
||||
quant_group_size, is_arch_support_pdl(), write_shared_sf, copy_topk
|
||||
)
|
||||
return load_jit(
|
||||
make_name("mega_moe_pre_dispatch"),
|
||||
*args,
|
||||
@@ -127,12 +131,13 @@ def mega_moe_pre_dispatch(
|
||||
*,
|
||||
shared_x_sf: Optional[torch.Tensor] = None,
|
||||
shared_block_m: int = 0,
|
||||
copy_topk: bool = True,
|
||||
) -> None:
|
||||
if x.shape[0] == 0 and x.stride(0) != x.shape[1]:
|
||||
x = x.new_empty(x.shape)
|
||||
# shared_block_m must match the consuming MoE's paged scale layout
|
||||
module = _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size, shared_x_sf is not None
|
||||
quant_group_size, shared_x_sf is not None, copy_topk
|
||||
)
|
||||
module.run(
|
||||
x,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def bf16_mega_gate(
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
top_k: int,
|
||||
*,
|
||||
routed_scaling_factor: float,
|
||||
ep_rank: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
image_bias: Optional[torch.Tensor] = None,
|
||||
image_token_mask: Optional[torch.Tensor] = None,
|
||||
valid_token_mask: Optional[torch.Tensor] = None,
|
||||
hash_topk_ids: Optional[torch.Tensor] = None,
|
||||
to_physical_map: Optional[torch.Tensor] = None,
|
||||
logical_count: Optional[torch.Tensor] = None,
|
||||
out: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import deep_gemm
|
||||
|
||||
if valid_token_mask is not None:
|
||||
hidden_states = torch.where(valid_token_mask[:, None], hidden_states, 0)
|
||||
|
||||
if out is None:
|
||||
ids = torch.empty(
|
||||
(hidden_states.shape[0], top_k),
|
||||
dtype=torch.int64,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
weights = torch.empty_like(ids, dtype=torch.float32)
|
||||
else:
|
||||
ids, weights = out
|
||||
deep_gemm.bf16_mega_gate(
|
||||
hidden_states,
|
||||
weight,
|
||||
top_k,
|
||||
use_shared_as_routed=False,
|
||||
num_shared_experts=0,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
ep_rank=ep_rank,
|
||||
scoring_func="sqrtsoftplus",
|
||||
mask=valid_token_mask,
|
||||
bias=bias,
|
||||
image_bias=image_bias,
|
||||
image_token_mask=image_token_mask,
|
||||
fix_routing_mask=(
|
||||
torch.ones_like(ids[:, 0], dtype=torch.bool)
|
||||
if hash_topk_ids is not None
|
||||
else None
|
||||
),
|
||||
unmapped_topk_idx=hash_topk_ids,
|
||||
to_physical_map=to_physical_map,
|
||||
logical_count=logical_count,
|
||||
out=(ids, weights),
|
||||
)
|
||||
return weights, ids
|
||||
@@ -1432,6 +1432,7 @@ class Envs:
|
||||
# residency margin so every cluster can launch beside other streams.
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2)
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True)
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_GATE = EnvBool(False)
|
||||
|
||||
# ===================================================================
|
||||
# Top-k kernels
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopKConfig,
|
||||
_post_process_topk_ids,
|
||||
apply_routing_simulation,
|
||||
routing_simulation_enabled,
|
||||
)
|
||||
from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec, get_forward, get_parallel
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
|
||||
# DeepGEMM bf16_mega_gate scores one token block per launch.
|
||||
_MEGA_GATE_MAX_ROWS = 2048
|
||||
|
||||
|
||||
class MegaGateRouting(NamedTuple):
|
||||
topk_weights: Optional[torch.Tensor]
|
||||
topk_ids: Optional[torch.Tensor]
|
||||
routed_scaling_factor: float
|
||||
in_buffer: bool
|
||||
|
||||
|
||||
def mega_gate_supported(moe) -> bool:
|
||||
if (
|
||||
_device_sm // 10 != 10
|
||||
or moe.config.model_type != "deepseek_v41"
|
||||
or moe.topk.enable_waterfill
|
||||
or moe.num_fused_shared_experts
|
||||
or get_exec().deterministic.enable_deterministic_inference
|
||||
):
|
||||
return False
|
||||
weight = moe.gate.weight
|
||||
num_experts, hidden = weight.shape
|
||||
top_k = moe.config.num_experts_per_tok
|
||||
if (
|
||||
weight.dtype != torch.bfloat16
|
||||
or not weight.is_contiguous()
|
||||
or hidden % 256 != 0
|
||||
or not 0 < num_experts <= 512
|
||||
or num_experts % 4 != 0
|
||||
or not 0 < top_k <= min(32, num_experts)
|
||||
):
|
||||
return False
|
||||
bias_vl = None if moe.is_hash else moe.gate.e_score_correction_bias_vl
|
||||
for bias in (moe.gate.e_score_correction_bias, bias_vl):
|
||||
if bias is not None and (
|
||||
bias.dtype != torch.float32
|
||||
or not bias.is_contiguous()
|
||||
or bias.shape != (num_experts,)
|
||||
):
|
||||
return False
|
||||
if moe.is_hash:
|
||||
return (
|
||||
moe.topk.score_func == "sqrtsoftplus"
|
||||
and moe.topk.num_shared_topk_slots == 0
|
||||
and moe.topk.topk == top_k
|
||||
)
|
||||
config = moe.topk.topk_config
|
||||
return (
|
||||
config.scoring_func == "sqrtsoftplus"
|
||||
and config.sqrtsoftplus_log1p
|
||||
and config.renormalize
|
||||
and config.num_shared_topk_slots == 0
|
||||
and config.top_k == top_k
|
||||
and not (top_k == 1 and bias_vl is not None)
|
||||
and not config.use_grouped_topk
|
||||
and config.custom_routing_function is None
|
||||
and not config.torch_native
|
||||
)
|
||||
|
||||
|
||||
def should_use_mega_gate(moe, hidden_states: torch.Tensor) -> bool:
|
||||
forward = get_forward()
|
||||
return (
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get()
|
||||
and moe.mega_gate_supported
|
||||
and 0 < hidden_states.shape[0] <= _MEGA_GATE_MAX_ROWS
|
||||
and not forward.is_extend_in_batch
|
||||
and not forward.sp_active
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and hidden_states.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def _captures_routed_experts(moe) -> bool:
|
||||
return (
|
||||
not moe.is_hash
|
||||
and moe.topk.topk_config.allow_routed_experts_capture
|
||||
and get_global_experts_capturer() is not None
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def _ones(num_experts: int, device: torch.device) -> torch.Tensor:
|
||||
return torch.ones(num_experts, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
def _physical_dispatch_maps(info) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if info.ep_dispatch_algorithm == "static":
|
||||
rank_map = info.partial_logical_to_rank_dispatch_physical_map
|
||||
return rank_map.to(torch.int32)[:, None], _ones(len(rank_map), rank_map.device)
|
||||
return (
|
||||
info.partial_logical_to_all_physical_map.to(torch.int32),
|
||||
info.partial_logical_to_all_physical_map_num_valid.to(torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateRouting]:
|
||||
bound, dispatch_info = moe.topk.routing_inputs()
|
||||
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm in (
|
||||
"lp",
|
||||
"fake",
|
||||
):
|
||||
return None
|
||||
if moe.is_hash and input_ids is None:
|
||||
return None
|
||||
num_tokens = hidden_states.shape[0]
|
||||
valid = None
|
||||
if bound is not None:
|
||||
valid = (
|
||||
torch.arange(num_tokens, device=hidden_states.device, dtype=torch.int32)
|
||||
< bound.reshape(-1)[:1]
|
||||
)
|
||||
image = hash_ids = None
|
||||
if moe.is_hash:
|
||||
token_ids = input_ids[:num_tokens]
|
||||
if valid is not None:
|
||||
token_ids = torch.where(valid, token_ids, 0)
|
||||
hash_ids = moe.topk.tid2eid[token_ids].to(torch.int64)
|
||||
elif moe.gate.e_score_correction_bias_vl is not None and input_ids is not None:
|
||||
image = (input_ids[:num_tokens] == moe.config.image_token_id).contiguous()
|
||||
|
||||
routed_scaling_factor = float(moe.routed_scaling_factor)
|
||||
fused_in_topk = moe.routed_scaling_factor_fused_in_topk
|
||||
# The benchmark override writes logical ids, so with EPLB it must run before the remap.
|
||||
in_buffer = (
|
||||
buf is not None
|
||||
and not _captures_routed_experts(moe)
|
||||
and not (dispatch_info is not None and routing_simulation_enabled())
|
||||
)
|
||||
to_physical_map = logical_count = None
|
||||
if in_buffer and dispatch_info is not None:
|
||||
# NOTE(kpham-sgl): MegaMoE reads physical expert slots from this buffer, so
|
||||
# under EPLB the gate must map logical ids in-kernel to stay on this path.
|
||||
to_physical_map, logical_count = _physical_dispatch_maps(dispatch_info)
|
||||
weights, ids = bf16_mega_gate(
|
||||
hidden_states,
|
||||
moe.gate.weight,
|
||||
moe.config.num_experts_per_tok,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if in_buffer or fused_in_topk else 1.0
|
||||
),
|
||||
ep_rank=get_parallel().moe_ep_rank,
|
||||
bias=moe.gate.e_score_correction_bias,
|
||||
image_bias=moe.gate.e_score_correction_bias_vl if image is not None else None,
|
||||
image_token_mask=image,
|
||||
valid_token_mask=valid,
|
||||
hash_topk_ids=hash_ids,
|
||||
to_physical_map=to_physical_map,
|
||||
logical_count=logical_count,
|
||||
out=(
|
||||
(buf.topk_idx[:num_tokens], buf.topk_weights[:num_tokens])
|
||||
if in_buffer
|
||||
else None
|
||||
),
|
||||
)
|
||||
if not moe.is_hash and routing_simulation_enabled():
|
||||
top_k = ids.shape[1]
|
||||
apply_routing_simulation(
|
||||
ids,
|
||||
weights,
|
||||
moe.gate.weight.shape[0],
|
||||
num_routed_topk=top_k,
|
||||
top_k=top_k,
|
||||
layer_id=moe.layer_id,
|
||||
num_token_non_padded=bound,
|
||||
weight_scale=(
|
||||
routed_scaling_factor if in_buffer and not fused_in_topk else 1.0
|
||||
),
|
||||
)
|
||||
if in_buffer:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=buf.topk_idx[:num_tokens]
|
||||
)
|
||||
return MegaGateRouting(None, None, 1.0, True)
|
||||
|
||||
if moe.is_hash:
|
||||
config = TopKConfig(
|
||||
top_k=moe.topk.topk,
|
||||
routed_scaling_factor=moe.topk.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=moe.topk.apply_routed_scaling_factor_on_output,
|
||||
allow_routed_experts_capture=False,
|
||||
)
|
||||
else:
|
||||
config = moe.topk.topk_config
|
||||
ids, weights, recorder_ids = _post_process_topk_ids(
|
||||
ids,
|
||||
weights,
|
||||
config,
|
||||
hidden_states.new_empty((0, moe.gate.weight.shape[0])),
|
||||
moe.layer_id,
|
||||
num_token_non_padded=bound,
|
||||
expert_location_dispatch_info=dispatch_info,
|
||||
padded_rows_masked=True,
|
||||
)
|
||||
if recorder_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_ids
|
||||
)
|
||||
return MegaGateRouting(
|
||||
weights, ids, 1.0 if fused_in_topk else routed_scaling_factor, False
|
||||
)
|
||||
@@ -25,6 +25,7 @@ from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.moe.mega_gate import run_mega_gate, should_use_mega_gate
|
||||
from sglang.srt.layers.moe.mega_moe_sm90 import (
|
||||
is_sm90_fp8_mega_moe_available,
|
||||
run_sm90_mega_routed,
|
||||
@@ -250,6 +251,25 @@ def forward_mega_moe(
|
||||
return y
|
||||
|
||||
|
||||
def _mega_moe_buffer(moe: DeepseekV2MoE):
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
return _get_mega_moe_symm_buffer(
|
||||
get_parallel().moe_ep_group.device_group,
|
||||
num_experts=moe.experts.num_experts,
|
||||
num_max_tokens_per_rank=(
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
|
||||
),
|
||||
num_topk=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
|
||||
hidden=moe.config.hidden_size,
|
||||
intermediate_hidden=moe.config.moe_intermediate_size,
|
||||
num_shared_experts=(
|
||||
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
|
||||
),
|
||||
mma_type=_mega_moe_mma_type(moe.experts),
|
||||
)
|
||||
|
||||
|
||||
def _run_mega_routed(
|
||||
moe: DeepseekV2MoE,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -259,7 +279,24 @@ def _run_mega_routed(
|
||||
) -> torch.Tensor:
|
||||
hidden_size = moe.config.hidden_size
|
||||
|
||||
if num_tokens > 0:
|
||||
routing = None
|
||||
if should_use_mega_gate(moe, hidden_states):
|
||||
import deep_gemm
|
||||
|
||||
# Only the fp8xfp4 pre-dispatch can skip its top-k copy (kCopyTopk).
|
||||
buf = (
|
||||
_mega_moe_buffer(moe)
|
||||
if _mega_moe_mma_type(moe.experts) == "fp8xfp4"
|
||||
else None
|
||||
)
|
||||
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
|
||||
routing = run_mega_gate(moe, hidden_states, input_ids_global, buf)
|
||||
|
||||
if routing is not None:
|
||||
topk_ids = routing.topk_ids
|
||||
topk_weights = routing.topk_weights
|
||||
routed_scaling_factor = routing.routed_scaling_factor
|
||||
elif num_tokens > 0:
|
||||
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
|
||||
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
|
||||
if (
|
||||
@@ -282,6 +319,12 @@ def _run_mega_routed(
|
||||
else:
|
||||
topk_ids = None
|
||||
topk_weights = None
|
||||
if routing is None:
|
||||
routed_scaling_factor = (
|
||||
1.0
|
||||
if moe.routed_scaling_factor_fused_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
)
|
||||
|
||||
return run_mega_routed_experts(
|
||||
moe.experts,
|
||||
@@ -293,16 +336,13 @@ def _run_mega_routed(
|
||||
top_k=moe.config.num_experts_per_tok + moe.num_fused_shared_experts,
|
||||
num_tokens=num_tokens,
|
||||
activation_clamp=moe.experts.moe_runner_config.swiglu_limit,
|
||||
routed_scaling_factor=(
|
||||
1.0
|
||||
if moe.routed_scaling_factor_fused_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
),
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
num_shared_experts=(
|
||||
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
|
||||
),
|
||||
shared_l1_weights=moe.mega_shared_l1_weights,
|
||||
shared_l2_weights=moe.mega_shared_l2_weights,
|
||||
routing_in_buffer=routing is not None and routing.in_buffer,
|
||||
)
|
||||
|
||||
|
||||
@@ -321,6 +361,7 @@ def run_mega_routed_experts(
|
||||
num_shared_experts: int = 0,
|
||||
shared_l1_weights=None,
|
||||
shared_l2_weights=None,
|
||||
routing_in_buffer: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Rows are this rank's tokens; the returned rows are fully combined.
|
||||
import deep_gemm
|
||||
@@ -351,7 +392,10 @@ def run_mega_routed_experts(
|
||||
mma_type=mma_type,
|
||||
)
|
||||
|
||||
if num_tokens > 0:
|
||||
if routing_in_buffer:
|
||||
assert num_tokens > 0 and _device_sm != 90 and mma_type == "fp8xfp4"
|
||||
topk_ids_in = topk_weights_in = None
|
||||
elif num_tokens > 0:
|
||||
topk_ids_in = topk_ids.to(torch.int32)
|
||||
topk_weights_in = topk_weights.to(torch.float32)
|
||||
else:
|
||||
@@ -372,6 +416,7 @@ def run_mega_routed_experts(
|
||||
)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
assert not routing_in_buffer
|
||||
topk_weights_in = topk_weights_in * routed_scaling_factor
|
||||
|
||||
mega_kwargs = {"recipe": (1, 1, 32)}
|
||||
@@ -429,8 +474,8 @@ def run_mega_routed_experts(
|
||||
)
|
||||
mega_moe_pre_dispatch(
|
||||
hidden_states,
|
||||
topk_ids_in,
|
||||
topk_weights_in,
|
||||
buf.topk_idx if routing_in_buffer else topk_ids_in,
|
||||
buf.topk_weights if routing_in_buffer else topk_weights_in,
|
||||
buf.x,
|
||||
buf.x_sf,
|
||||
buf.topk_idx,
|
||||
@@ -440,6 +485,7 @@ def run_mega_routed_experts(
|
||||
buf.shared_l1_acts_sf if shared_l1_weights is not None else None
|
||||
),
|
||||
shared_block_m=shared_block_m,
|
||||
copy_topk=not routing_in_buffer,
|
||||
)
|
||||
|
||||
# Allocate at least one row so y has a non-null CUDA data_ptr;
|
||||
|
||||
@@ -467,9 +467,11 @@ def _simulate_balanced_routing_kernel(
|
||||
stride_ik,
|
||||
stride_wm,
|
||||
stride_wk,
|
||||
num_non_padded_ptr,
|
||||
K: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
RANDOM: tl.constexpr,
|
||||
HAS_NON_PADDED: tl.constexpr = False,
|
||||
):
|
||||
"""One program per token: overwrite its top-k row with a balanced expert
|
||||
assignment and uniform ``1/k`` weights, in a single launch — so the
|
||||
@@ -505,12 +507,13 @@ def _simulate_balanced_routing_kernel(
|
||||
gid = (deal % num_ranks) * experts_per_rank + (
|
||||
deal // num_ranks
|
||||
) % experts_per_rank
|
||||
w = tl.full((BLOCK_K,), inv_k, tl.float32)
|
||||
if HAS_NON_PADDED:
|
||||
is_pad = t >= tl.load(num_non_padded_ptr)
|
||||
gid = tl.where(is_pad, -1, gid)
|
||||
w = tl.where(is_pad, 0.0, w)
|
||||
tl.store(topk_ids_ptr + t * stride_im + j * stride_ik, gid, mask=mask)
|
||||
tl.store(
|
||||
topk_weights_ptr + t * stride_wm + j * stride_wk,
|
||||
tl.full((BLOCK_K,), inv_k, tl.float32),
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(topk_weights_ptr + t * stride_wm + j * stride_wk, w, mask=mask)
|
||||
|
||||
|
||||
# Per-launch seed for the uniform (RANDOM=True) path: varies across eager calls so
|
||||
@@ -529,6 +532,8 @@ def _simulate_balanced_routing(
|
||||
token_shard_rank: int = 0,
|
||||
num_token_shards: int = 1,
|
||||
seed: Optional[int] = None,
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
weight_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Benchmark-only fused override (in place): replace ``topk_ids`` with a
|
||||
balanced expert assignment and ``topk_weights`` with ``1/k`` using a single
|
||||
@@ -589,7 +594,7 @@ def _simulate_balanced_routing(
|
||||
num_ranks,
|
||||
experts_per_rank,
|
||||
deal_offset,
|
||||
1.0 / k,
|
||||
weight_scale / k,
|
||||
seed,
|
||||
token_shard_rank,
|
||||
num_token_shards,
|
||||
@@ -597,9 +602,11 @@ def _simulate_balanced_routing(
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
num_token_non_padded if num_token_non_padded is not None else topk_ids,
|
||||
K=k,
|
||||
BLOCK_K=triton.next_power_of_2(k),
|
||||
RANDOM=random,
|
||||
HAS_NON_PADDED=num_token_non_padded is not None,
|
||||
)
|
||||
|
||||
|
||||
@@ -2401,6 +2408,8 @@ def apply_routing_simulation(
|
||||
num_routed_topk: int,
|
||||
top_k: int,
|
||||
layer_id: Optional[int],
|
||||
num_token_non_padded: Optional[torch.Tensor] = None,
|
||||
weight_scale: float = 1.0,
|
||||
) -> None:
|
||||
simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
# Benchmark-only: override gating with a balanced expert assignment (so
|
||||
@@ -2441,6 +2450,8 @@ def apply_routing_simulation(
|
||||
layer_id=layer_id,
|
||||
token_shard_rank=token_shard_rank,
|
||||
num_token_shards=num_token_shards,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
weight_scale=weight_scale,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1869,6 +1869,12 @@ class DeepseekV2MoE(nn.Module):
|
||||
def _routed_mxfp8_prequant_static_enabled(self) -> bool:
|
||||
return self._compute_routed_mxfp8_prequant_enabled()[0]
|
||||
|
||||
@cached_property
|
||||
def mega_gate_supported(self) -> bool:
|
||||
from sglang.srt.layers.moe.mega_gate import mega_gate_supported
|
||||
|
||||
return mega_gate_supported(self)
|
||||
|
||||
def _compute_routed_mxfp8_prequant_enabled(self) -> Tuple[bool, str]:
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatcher
|
||||
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32
|
||||
from sglang.kernels.ops.moe.mega_gate import bf16_mega_gate
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
class TestMegaGate(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
raise unittest.SkipTest("MegaGate requires SM10x")
|
||||
|
||||
@staticmethod
|
||||
def _reference(
|
||||
x, weight, top_k, bias, image_bias, image_mask, valid_mask, hash_ids
|
||||
):
|
||||
x = torch.where(valid_mask[:, None], x, 0)
|
||||
scores = torch.log1p(torch.exp(linear_bf16_fp32(x, weight))).sqrt()
|
||||
if hash_ids is not None:
|
||||
ids = hash_ids
|
||||
else:
|
||||
row_bias = (
|
||||
torch.where(image_mask[:, None], image_bias, bias)
|
||||
if image_mask is not None
|
||||
else bias
|
||||
)
|
||||
ids = (scores + row_bias).topk(top_k, dim=-1).indices
|
||||
weights = scores.gather(1, ids)
|
||||
weights = weights / (weights.sum(-1, keepdim=True) + 1e-20) * 1.5
|
||||
return weights.masked_fill(~valid_mask[:, None], 0), ids.masked_fill(
|
||||
~valid_mask[:, None], -1
|
||||
)
|
||||
|
||||
def _assert_routes_close(self, actual, expected):
|
||||
actual_weights, actual_ids = actual
|
||||
expected_weights, expected_ids = expected
|
||||
actual_ids, order = actual_ids.sort(dim=-1)
|
||||
actual_weights = actual_weights.gather(1, order)
|
||||
expected_ids, order = expected_ids.sort(dim=-1)
|
||||
expected_weights = expected_weights.gather(1, order)
|
||||
torch.testing.assert_close(actual_ids, expected_ids)
|
||||
torch.testing.assert_close(
|
||||
actual_weights, expected_weights, rtol=1e-3, atol=1e-4
|
||||
)
|
||||
|
||||
def test_routing_and_cuda_graph_replay(self):
|
||||
torch.manual_seed(42)
|
||||
for num_experts, top_k in ((384, 6), (128, 6)):
|
||||
for tokens in (1, 17, 512):
|
||||
for routing in ("text", "image", "hash"):
|
||||
with self.subTest(
|
||||
num_experts=num_experts, tokens=tokens, routing=routing
|
||||
):
|
||||
x = torch.randn(
|
||||
(tokens, 5120), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
weight = (
|
||||
torch.randn(
|
||||
(num_experts, 5120), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
* 0.001
|
||||
)
|
||||
bias = torch.arange(num_experts, device="cuda").float() * 0.25
|
||||
image_bias = bias.flip(0) if routing == "image" else None
|
||||
image_mask = (
|
||||
torch.zeros(tokens, device="cuda", dtype=torch.bool)
|
||||
if routing == "image"
|
||||
else None
|
||||
)
|
||||
hash_ids = (
|
||||
torch.arange(top_k, device="cuda")
|
||||
.expand(tokens, -1)
|
||||
.clone()
|
||||
if routing == "hash"
|
||||
else None
|
||||
)
|
||||
valid_mask = torch.ones(tokens, device="cuda", dtype=torch.bool)
|
||||
|
||||
def run():
|
||||
return bf16_mega_gate(
|
||||
x,
|
||||
weight,
|
||||
top_k,
|
||||
routed_scaling_factor=1.5,
|
||||
ep_rank=0,
|
||||
bias=bias,
|
||||
image_bias=image_bias,
|
||||
image_token_mask=image_mask,
|
||||
valid_token_mask=valid_mask,
|
||||
hash_topk_ids=hash_ids,
|
||||
)
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
run()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
actual = run()
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
for valid_rows, padding in (
|
||||
(tokens, float("nan")),
|
||||
(tokens // 2, float("nan")),
|
||||
(0, float("inf")),
|
||||
(tokens, float("nan")),
|
||||
):
|
||||
x.normal_()
|
||||
valid_mask.copy_(
|
||||
torch.arange(tokens, device="cuda") < valid_rows
|
||||
)
|
||||
x[valid_rows:] = padding
|
||||
if image_mask is not None:
|
||||
image_mask.logical_not_()
|
||||
if hash_ids is not None:
|
||||
hash_ids.copy_(
|
||||
torch.arange(top_k, device="cuda")
|
||||
.expand(tokens, -1)
|
||||
.roll(1, dims=-1)
|
||||
)
|
||||
expected = self._reference(
|
||||
x,
|
||||
weight,
|
||||
top_k,
|
||||
bias,
|
||||
image_bias,
|
||||
image_mask,
|
||||
valid_mask,
|
||||
hash_ids,
|
||||
)
|
||||
graph.replay()
|
||||
self._assert_routes_close(actual, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -132,6 +132,27 @@ def test_kernel_matches_reference_on_every_shard(
|
||||
assert torch.equal(weights, torch.full_like(weights, 1.0 / topk))
|
||||
|
||||
|
||||
def test_kernel_masks_padded_rows_and_scales_weights():
|
||||
tokens, real, scale = 300, 250, 1.5
|
||||
ids = torch.zeros((tokens, TOPK), dtype=torch.int64, device="cuda")
|
||||
weights = torch.ones((tokens, TOPK), dtype=torch.float32, device="cuda")
|
||||
_simulate_balanced_routing(
|
||||
ids,
|
||||
weights,
|
||||
NUM_EXPERTS,
|
||||
random=False,
|
||||
num_ranks=EP_SIZE,
|
||||
layer_id=3,
|
||||
num_token_non_padded=torch.tensor([real], device="cuda"),
|
||||
weight_scale=scale,
|
||||
)
|
||||
expected_ids, expected_weights = _route(tokens, layer_id=3)
|
||||
assert torch.equal(ids[:real], expected_ids[:real].long())
|
||||
torch.testing.assert_close(weights[:real], expected_weights[:real] * scale)
|
||||
assert torch.equal(ids[real:], torch.full_like(ids[real:], -1))
|
||||
assert torch.equal(weights[real:], torch.zeros_like(weights[real:]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 300])
|
||||
@pytest.mark.parametrize("num_ranks", [4, 8, 16])
|
||||
def test_single_shard_balances_within_one_slot(num_tokens, num_ranks):
|
||||
|
||||
@@ -2,19 +2,23 @@
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from contextlib import ExitStack, contextmanager, nullcontext
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend, mega_moe
|
||||
from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend
|
||||
from sglang.srt.layers.moe import mega_gate as mega_gate_runtime
|
||||
from sglang.srt.layers.moe import mega_moe
|
||||
from sglang.srt.layers.moe import topk as topk_module
|
||||
from sglang.srt.layers.moe.fused_moe_triton import layer as fused_moe_layer_module
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
@@ -450,6 +454,219 @@ class TestDeepGemmMegaMoeApi(CustomTestCase):
|
||||
self.assertEqual(out.shape, (0, 4))
|
||||
moe.gate.assert_not_called()
|
||||
|
||||
@staticmethod
|
||||
def _mega_gate_reference(x, weight, top_k, **kwargs):
|
||||
if not torch.isfinite(x).all():
|
||||
raise ValueError("MegaGate rejects nonfinite GEMM inputs")
|
||||
scores = torch.log1p(torch.exp(x.float() @ weight.float().T)).sqrt()
|
||||
if kwargs["unmapped_topk_idx"] is not None:
|
||||
ids = kwargs["unmapped_topk_idx"]
|
||||
else:
|
||||
bias = kwargs["bias"]
|
||||
if kwargs["image_bias"] is not None:
|
||||
bias = torch.where(
|
||||
kwargs["image_token_mask"][:, None], kwargs["image_bias"], bias
|
||||
)
|
||||
ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
weights = scores.gather(1, ids)
|
||||
weights = weights / (weights.sum(-1, keepdim=True) + 1e-20)
|
||||
weights = weights * kwargs["routed_scaling_factor"]
|
||||
if kwargs["to_physical_map"] is not None:
|
||||
physical_map, count = kwargs["to_physical_map"], kwargs["logical_count"]
|
||||
assert physical_map.dtype == count.dtype == torch.int32
|
||||
token = torch.arange(ids.shape[0])[:, None]
|
||||
replica = (kwargs["ep_rank"] + token * 23333) % count[ids]
|
||||
ids = physical_map[ids, replica]
|
||||
if kwargs["mask"] is not None:
|
||||
weights = weights.masked_fill(~kwargs["mask"][:, None], 0)
|
||||
ids = ids.masked_fill(~kwargs["mask"][:, None], -1)
|
||||
kwargs["out"][0].copy_(ids)
|
||||
kwargs["out"][1].copy_(weights)
|
||||
|
||||
def _mega_gate_moe(self, scale_in_topk=False, hash_routing=False):
|
||||
moe = self._v41_moe(0, scale_in_topk)
|
||||
moe.config.model_type = "deepseek_v41"
|
||||
moe.config.hidden_size = 256
|
||||
moe.gate.weight = torch.zeros((8, 256), dtype=torch.bfloat16)
|
||||
moe.gate.e_score_correction_bias = torch.tensor(
|
||||
[4.0, 3.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
|
||||
)
|
||||
moe.gate.e_score_correction_bias_vl = torch.tensor(
|
||||
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 4.0]
|
||||
)
|
||||
moe.topk.topk_config.sqrtsoftplus_log1p = True
|
||||
moe.topk.enable_waterfill = False
|
||||
moe.mega_gate_supported = True
|
||||
if hash_routing:
|
||||
moe.is_hash = True
|
||||
moe.gate.e_score_correction_bias = None
|
||||
moe.topk = SimpleNamespace(
|
||||
enable_waterfill=False,
|
||||
topk=2,
|
||||
num_shared_topk_slots=0,
|
||||
routed_scaling_factor=2.0,
|
||||
apply_routed_scaling_factor_on_output=scale_in_topk,
|
||||
score_func="sqrtsoftplus",
|
||||
tid2eid=torch.tensor([[6, 7]] * 100, dtype=torch.int32),
|
||||
)
|
||||
return moe
|
||||
|
||||
@contextmanager
|
||||
def _mega_gate_env(self):
|
||||
deep_gemm = ModuleType("deep_gemm")
|
||||
deep_gemm.bf16_mega_gate = self._mega_gate_reference
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(
|
||||
get_context().override_server_args(
|
||||
model_path="dummy", enable_deterministic_inference=False
|
||||
)
|
||||
)
|
||||
stack.enter_context(get_parallel().override(moe_ep_rank=0))
|
||||
stack.enter_context(patch.dict(sys.modules, {"deep_gemm": deep_gemm}))
|
||||
stack.enter_context(patch.object(topk_module, "_is_cuda", True))
|
||||
stack.enter_context(
|
||||
patch.object(topk_module, "_can_fuse_padded_region", return_value=False)
|
||||
)
|
||||
yield
|
||||
|
||||
def test_mega_gate_admission_falls_back_for_unsupported_routers(self):
|
||||
moe = self._mega_gate_moe()
|
||||
supported = mega_gate_runtime.mega_gate_supported
|
||||
with (
|
||||
self._mega_gate_env(),
|
||||
patch.object(mega_gate_runtime, "_device_sm", 100),
|
||||
):
|
||||
self.assertTrue(supported(moe))
|
||||
self.assertTrue(supported(self._mega_gate_moe(hash_routing=True)))
|
||||
with patch.object(moe.topk.topk_config, "scoring_func", "sigmoid"):
|
||||
self.assertFalse(supported(moe))
|
||||
with patch.object(moe, "num_fused_shared_experts", 1):
|
||||
self.assertFalse(supported(moe))
|
||||
with get_exec().deterministic.override(enable_deterministic_inference=True):
|
||||
self.assertFalse(supported(moe))
|
||||
|
||||
hidden = torch.zeros((32, 256), dtype=torch.bfloat16)
|
||||
with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(False):
|
||||
self.assertFalse(mega_gate_runtime.should_use_mega_gate(moe, hidden))
|
||||
with envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.override(True):
|
||||
self.assertTrue(mega_gate_runtime.should_use_mega_gate(moe, hidden))
|
||||
self.assertFalse(
|
||||
mega_gate_runtime.should_use_mega_gate(moe, hidden.float())
|
||||
)
|
||||
self.assertFalse(
|
||||
mega_gate_runtime.should_use_mega_gate(
|
||||
moe, torch.zeros((4096, 256), dtype=torch.bfloat16)
|
||||
)
|
||||
)
|
||||
|
||||
def test_mega_gate_routes_vision_hash_and_padding(self):
|
||||
for is_hash in (False, True):
|
||||
for scale_in_topk in (False, True):
|
||||
for in_buffer in (False, True):
|
||||
with self.subTest(
|
||||
is_hash=is_hash,
|
||||
scale_in_topk=scale_in_topk,
|
||||
in_buffer=in_buffer,
|
||||
):
|
||||
moe = self._mega_gate_moe(scale_in_topk, is_hash)
|
||||
hidden = torch.ones((4, 256), dtype=torch.bfloat16)
|
||||
hidden[3:] = float("nan")
|
||||
input_ids = torch.tensor([7, 99, 7, 1000])
|
||||
buf = SimpleNamespace(
|
||||
topk_idx=torch.full((8, 2), 5, dtype=torch.int64),
|
||||
topk_weights=torch.full((8, 2), 9.0),
|
||||
)
|
||||
with (
|
||||
self._mega_gate_env(),
|
||||
patch.object(
|
||||
moe.topk,
|
||||
"routing_inputs",
|
||||
return_value=(torch.tensor([3]), None),
|
||||
create=True,
|
||||
),
|
||||
):
|
||||
routing = mega_gate_runtime.run_mega_gate(
|
||||
moe, hidden, input_ids, buf if in_buffer else None
|
||||
)
|
||||
self.assertEqual(routing.in_buffer, in_buffer)
|
||||
if in_buffer:
|
||||
ids, weights = buf.topk_idx[:4], buf.topk_weights[:4]
|
||||
self.assertEqual(routing.routed_scaling_factor, 1.0)
|
||||
self.assertTrue((buf.topk_idx[4:] == 5).all())
|
||||
scale = 2.0
|
||||
else:
|
||||
ids, weights = routing.topk_ids, routing.topk_weights
|
||||
self.assertEqual(
|
||||
routing.routed_scaling_factor,
|
||||
1.0 if scale_in_topk else 2.0,
|
||||
)
|
||||
scale = 2.0 if scale_in_topk else 1.0
|
||||
image = [6, 7] if is_hash else [7, 6]
|
||||
text = [6, 7] if is_hash else [0, 1]
|
||||
torch.testing.assert_close(
|
||||
ids,
|
||||
torch.tensor([text, image, text, [-1, -1]]),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights,
|
||||
torch.tensor([[0.5 * scale] * 2] * 3 + [[0.0, 0.0]]),
|
||||
)
|
||||
|
||||
def test_mega_gate_maps_eplb_in_kernel_and_records_from_buffer(self):
|
||||
moe = self._mega_gate_moe()
|
||||
hidden = torch.ones((4, 256), dtype=torch.bfloat16)
|
||||
hidden[2:] = float("nan")
|
||||
dispatch_info = ExpertLocationDispatchInfo(
|
||||
ep_dispatch_algorithm="static",
|
||||
partial_logical_to_rank_dispatch_physical_map=torch.tensor(
|
||||
[9, 8, 7, 6, 5, 4, 3, 2]
|
||||
),
|
||||
partial_logical_to_all_physical_map=None,
|
||||
partial_logical_to_all_physical_map_num_valid=None,
|
||||
num_physical_experts=10,
|
||||
)
|
||||
recorded = []
|
||||
recorder = SimpleNamespace(
|
||||
on_select_experts=lambda **kw: recorded.append(kw["topk_ids"].clone())
|
||||
)
|
||||
buf = SimpleNamespace(
|
||||
topk_idx=torch.zeros((8, 2), dtype=torch.int64),
|
||||
topk_weights=torch.zeros((8, 2)),
|
||||
)
|
||||
with (
|
||||
self._mega_gate_env(),
|
||||
patch.object(
|
||||
mega_gate_runtime,
|
||||
"get_global_expert_distribution_recorder",
|
||||
return_value=recorder,
|
||||
),
|
||||
patch.object(
|
||||
moe.topk,
|
||||
"routing_inputs",
|
||||
return_value=(torch.tensor([2]), dispatch_info),
|
||||
),
|
||||
):
|
||||
routing = mega_gate_runtime.run_mega_gate(
|
||||
moe, hidden, torch.tensor([7, 99, 1000, 1000]), buf
|
||||
)
|
||||
self.assertTrue(routing.in_buffer)
|
||||
torch.testing.assert_close(
|
||||
buf.topk_idx[:4],
|
||||
torch.tensor([[9, 8], [2, 3], [-1, -1], [-1, -1]]),
|
||||
)
|
||||
self.assertTrue((buf.topk_weights[2:4] == 0).all())
|
||||
self.assertEqual(len(recorded), 1)
|
||||
torch.testing.assert_close(recorded[0], buf.topk_idx[:4])
|
||||
for algorithm in ("lp", "fake"):
|
||||
with patch.object(
|
||||
moe.topk,
|
||||
"routing_inputs",
|
||||
return_value=(None, SimpleNamespace(ep_dispatch_algorithm=algorithm)),
|
||||
):
|
||||
self.assertIsNone(
|
||||
mega_gate_runtime.run_mega_gate(moe, hidden, None, buf)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _v41_moe(fused_shared, scale_in_topk):
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user