fuse shared experts with routed experts in MegaMoE's DeepGEMM (#39313)
Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
co-authored by
Brayden Zhong
parent
c9cb32e310
commit
0e586fd12d
@@ -31,8 +31,11 @@ struct MegaMoEPreDispatchParams {
|
||||
int32_t* __restrict__ buf_x_sf; // row-major int32 [P, G/4]
|
||||
int64_t* __restrict__ buf_topk_idx; // [padded_max, top_k]
|
||||
float* __restrict__ buf_topk_weights; // [padded_max, top_k]
|
||||
int32_t* __restrict__ shared_x_sf; // column-major, BLOCK_M-paged scales
|
||||
|
||||
uint64_t buf_x_sf_stride_bytes;
|
||||
uint64_t shared_x_sf_col_stride;
|
||||
uint32_t shared_block_m;
|
||||
uint32_t num_tokens;
|
||||
uint32_t padded_max;
|
||||
uint32_t hidden;
|
||||
@@ -44,7 +47,7 @@ struct MegaMoEPreDispatchParams {
|
||||
// kMultiChunk covers rows too wide for one chunk per thread. It is a template
|
||||
// parameter rather than a runtime check because this kernel is issue-bound: the
|
||||
// loop bookkeeping alone costs ~10% on the rows that fit a single chunk.
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk>
|
||||
template <uint32_t kGroupSize, bool kUsePDL, bool kMultiChunk, bool kWriteSharedSF>
|
||||
__global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) {
|
||||
using namespace device;
|
||||
@@ -66,6 +69,12 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
const uint32_t token_id = bid;
|
||||
const auto token_in = params.x + static_cast<uint64_t>(token_id) * params.hidden;
|
||||
const auto token_out = params.buf_x + static_cast<uint64_t>(token_id) * params.hidden;
|
||||
uint32_t shared_row = 0;
|
||||
if constexpr (kWriteSharedSF) {
|
||||
const uint32_t m = token_id % params.shared_block_m;
|
||||
shared_row = token_id / params.shared_block_m * ((params.shared_block_m + 127u) / 128u * 128u) + m / 128u * 128u +
|
||||
m % 32u * 4u + m % 128u / 32u;
|
||||
}
|
||||
|
||||
const auto quantize_chunk = [&](uint32_t chunk) {
|
||||
InputVec in_vec;
|
||||
@@ -104,6 +113,11 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
if (within_group_id == 0 && group_id < params.num_groups) {
|
||||
const uint64_t byte_off = static_cast<uint64_t>(token_id) * params.buf_x_sf_stride_bytes + group_id;
|
||||
reinterpret_cast<uint8_t*>(params.buf_x_sf)[byte_off] = static_cast<uint8_t>(ue8m0_exp);
|
||||
if constexpr (kWriteSharedSF) {
|
||||
const uint64_t off = shared_row + static_cast<uint64_t>(group_id / 4u) * params.shared_x_sf_col_stride;
|
||||
reinterpret_cast<uint8_t*>(params.shared_x_sf)[off * sizeof(int32_t) + group_id % 4u] =
|
||||
static_cast<uint8_t>(ue8m0_exp);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -144,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void //
|
||||
// ---- Host wrapper
|
||||
// ------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
template <int64_t kGroupSize, bool kUsePDL>
|
||||
template <int64_t kGroupSize, bool kUsePDL, bool kWriteSharedSF>
|
||||
struct MegaMoEPreDispatchKernel {
|
||||
static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size");
|
||||
static constexpr auto kernel_one_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, false, kWriteSharedSF>;
|
||||
static constexpr auto kernel_multi_chunk =
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true>;
|
||||
mega_moe_pre_dispatch_kernel<static_cast<uint32_t>(kGroupSize), kUsePDL, true, kWriteSharedSF>;
|
||||
|
||||
static void
|
||||
run(const tvm::ffi::TensorView x,
|
||||
@@ -159,7 +173,9 @@ struct MegaMoEPreDispatchKernel {
|
||||
const tvm::ffi::TensorView buf_x,
|
||||
const tvm::ffi::TensorView buf_x_sf,
|
||||
const tvm::ffi::TensorView buf_topk_idx,
|
||||
const tvm::ffi::TensorView buf_topk_weights) {
|
||||
const tvm::ffi::TensorView buf_topk_weights,
|
||||
const tvm::ffi::Optional<tvm::ffi::TensorView> shared_x_sf,
|
||||
const int64_t shared_block_m) {
|
||||
using namespace host;
|
||||
|
||||
auto device = SymbolicDevice{};
|
||||
@@ -232,6 +248,21 @@ struct MegaMoEPreDispatchKernel {
|
||||
"hidden above 8192 must be a multiple of 256 so the tail quant chunks fill whole warps");
|
||||
RuntimeCheck(block_size >= top_k, "top_k must fit into one quant CTA");
|
||||
|
||||
int32_t* shared_sf_ptr = nullptr;
|
||||
uint64_t shared_sf_stride = 0;
|
||||
if constexpr (kWriteSharedSF) {
|
||||
RuntimeCheck(kGroupSize == 32, "shared MegaMoE scales require group_size=32");
|
||||
RuntimeCheck(shared_block_m > 0, "shared_block_m must be positive");
|
||||
auto S = SymbolicSize{"shared_sf_rows"};
|
||||
const auto sf = shared_x_sf.value();
|
||||
TensorMatcher({S, G4}).with_strides({1, -1}).with_dtype<int32_t>().with_device(device).verify(sf);
|
||||
const auto rows = (num_tokens + shared_block_m - 1) / shared_block_m * ((shared_block_m + 127) / 128 * 128);
|
||||
RuntimeCheck(S.unwrap() >= rows, "shared scale buffer is too small for shared_block_m");
|
||||
RuntimeCheck(sf.stride(1) >= S.unwrap(), "shared scale columns must not overlap");
|
||||
shared_sf_ptr = static_cast<int32_t*>(sf.data_ptr());
|
||||
shared_sf_stride = sf.stride(1);
|
||||
}
|
||||
|
||||
const auto pad_slots = (padded_max - num_tokens) * top_k;
|
||||
const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size);
|
||||
const auto num_total_blocks = num_tokens + num_pad_blocks;
|
||||
@@ -244,7 +275,10 @@ struct MegaMoEPreDispatchKernel {
|
||||
.buf_x_sf = static_cast<int32_t*>(buf_x_sf.data_ptr()),
|
||||
.buf_topk_idx = static_cast<int64_t*>(buf_topk_idx.data_ptr()),
|
||||
.buf_topk_weights = static_cast<float*>(buf_topk_weights.data_ptr()),
|
||||
.shared_x_sf = shared_sf_ptr,
|
||||
.buf_x_sf_stride_bytes = buf_x_sf_stride_bytes,
|
||||
.shared_x_sf_col_stride = shared_sf_stride,
|
||||
.shared_block_m = static_cast<uint32_t>(shared_block_m),
|
||||
.num_tokens = num_tokens,
|
||||
.padded_max = padded_max,
|
||||
.hidden = hidden,
|
||||
|
||||
@@ -39,8 +39,8 @@ def _jit_hash_topk_module():
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int):
|
||||
args = make_cpp_args(quant_group_size, is_arch_support_pdl())
|
||||
def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool):
|
||||
args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf)
|
||||
return load_jit(
|
||||
make_name("mega_moe_pre_dispatch"),
|
||||
*args,
|
||||
@@ -162,8 +162,14 @@ def mega_moe_pre_dispatch(
|
||||
buf_topk_idx: torch.Tensor,
|
||||
buf_topk_weights: torch.Tensor,
|
||||
quant_group_size: int = 32,
|
||||
*,
|
||||
shared_x_sf: Optional[torch.Tensor] = None,
|
||||
shared_block_m: int = 0,
|
||||
) -> None:
|
||||
module = _jit_mega_moe_pre_dispatch_module(quant_group_size)
|
||||
# shared_block_m must match the consuming MoE's paged scale layout
|
||||
module = _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size, shared_x_sf is not None
|
||||
)
|
||||
module.run(
|
||||
x,
|
||||
topk_idx,
|
||||
@@ -172,6 +178,8 @@ def mega_moe_pre_dispatch(
|
||||
buf_x_sf,
|
||||
buf_topk_idx,
|
||||
buf_topk_weights,
|
||||
shared_x_sf,
|
||||
shared_block_m,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1288,6 +1288,7 @@ class Envs:
|
||||
# Blackwell MegaMoE uses a whole-grid software barrier. Keep a small
|
||||
# residency margin so every cluster can launch beside other streams.
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2)
|
||||
SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True)
|
||||
|
||||
# ===================================================================
|
||||
# Top-k kernels
|
||||
|
||||
@@ -34,7 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils import is_hip, is_sm100_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deep_gemm import SymmBuffer
|
||||
@@ -117,6 +117,7 @@ def _get_mega_moe_symm_buffer(
|
||||
num_topk: int,
|
||||
hidden: int,
|
||||
intermediate_hidden: int,
|
||||
num_shared_experts: int = 0,
|
||||
mma_type: Optional[str] = None,
|
||||
) -> SymmBuffer:
|
||||
import deep_gemm
|
||||
@@ -132,6 +133,7 @@ def _get_mega_moe_symm_buffer(
|
||||
hidden,
|
||||
intermediate_hidden,
|
||||
mma_type,
|
||||
num_shared_experts,
|
||||
deep_gemm.get_num_sms(),
|
||||
)
|
||||
buf = _MEGA_MOE_SYMM_BUFFER.get(key)
|
||||
@@ -143,6 +145,7 @@ def _get_mega_moe_symm_buffer(
|
||||
num_topk,
|
||||
hidden,
|
||||
intermediate_hidden,
|
||||
num_shared_experts=num_shared_experts,
|
||||
mma_type=mma_type,
|
||||
activation="swiglu",
|
||||
)
|
||||
@@ -183,6 +186,19 @@ def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool
|
||||
return max_tokens_per_rank <= cap
|
||||
|
||||
|
||||
def should_fuse_mega_moe_shared_experts(moe: DeepseekV2MoE) -> bool:
|
||||
# DeepGEMM's in-kernel shared expert takes FP8 weights on SM100 fp8xfp4 only.
|
||||
return (
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS.get()
|
||||
and get_moe_a2a_backend().is_megamoe()
|
||||
and getattr(moe.experts, "_mega_moe_weights_built", False)
|
||||
and is_sm100_supported()
|
||||
and _mega_moe_mma_type(moe.experts) == "fp8xfp4"
|
||||
and moe.shared_experts_is_fp8
|
||||
and moe.shared_experts_weight_block_size == [128, 128]
|
||||
)
|
||||
|
||||
|
||||
def forward_mega_moe(
|
||||
moe: DeepseekV2MoE,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -199,29 +215,29 @@ def forward_mega_moe(
|
||||
)
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
|
||||
sbo_overlap_flag = (
|
||||
fused_shared = moe.mega_shared_l1_weights is not None
|
||||
fork_mega_moe = (
|
||||
moe.alt_stream is not None
|
||||
and moe.num_fused_shared_experts == 0
|
||||
and (fused_shared or moe.num_fused_shared_experts == 0)
|
||||
and num_tokens > 0
|
||||
and get_is_capture_mode()
|
||||
)
|
||||
|
||||
if sbo_overlap_flag:
|
||||
if fork_mega_moe:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
moe.alt_stream.wait_stream(current_stream)
|
||||
shared_output = moe._forward_shared_experts(hidden_states)
|
||||
mega_stream_ctx = torch.cuda.stream(moe.alt_stream)
|
||||
else:
|
||||
shared_output = moe._forward_shared_experts(hidden_states)
|
||||
mega_stream_ctx = nullcontext()
|
||||
|
||||
shared_output = None if fused_shared else moe._forward_shared_experts(hidden_states)
|
||||
|
||||
with mega_stream_ctx:
|
||||
y = _run_mega_routed(
|
||||
moe, hidden_states, forward_batch, input_ids_global, num_tokens
|
||||
)
|
||||
|
||||
if sbo_overlap_flag:
|
||||
if fork_mega_moe:
|
||||
current_stream.wait_stream(moe.alt_stream)
|
||||
|
||||
if shared_output is not None:
|
||||
@@ -275,6 +291,11 @@ def _run_mega_routed(
|
||||
if moe.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
else float(moe.routed_scaling_factor)
|
||||
),
|
||||
num_shared_experts=(
|
||||
0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts
|
||||
),
|
||||
shared_l1_weights=moe.mega_shared_l1_weights,
|
||||
shared_l2_weights=moe.mega_shared_l2_weights,
|
||||
)
|
||||
|
||||
|
||||
@@ -290,6 +311,9 @@ def run_mega_routed_experts(
|
||||
num_tokens: int,
|
||||
activation_clamp: Optional[float] = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
num_shared_experts: int = 0,
|
||||
shared_l1_weights=None,
|
||||
shared_l2_weights=None,
|
||||
) -> torch.Tensor:
|
||||
# Rows are this rank's tokens; the returned rows are fully combined.
|
||||
import deep_gemm
|
||||
@@ -316,6 +340,7 @@ def run_mega_routed_experts(
|
||||
num_topk=top_k,
|
||||
hidden=hidden_size,
|
||||
intermediate_hidden=intermediate_size,
|
||||
num_shared_experts=num_shared_experts,
|
||||
mma_type=mma_type,
|
||||
)
|
||||
|
||||
@@ -339,6 +364,9 @@ def run_mega_routed_experts(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights_in = topk_weights_in * routed_scaling_factor
|
||||
|
||||
mega_kwargs = {"recipe": (1, 1, 32)}
|
||||
if mma_type == "nvfp4xnvfp4":
|
||||
# Per-token outer scales go to buf.x_scales; the kernel folds them with
|
||||
@@ -380,6 +408,18 @@ def run_mega_routed_experts(
|
||||
mma_type=mma_type,
|
||||
)
|
||||
else:
|
||||
shared_block_m = 0
|
||||
if shared_l1_weights is not None:
|
||||
shared_block_m = deep_gemm.get_block_m_for_mega_moe(
|
||||
num_ranks=buf.group.size(),
|
||||
num_experts=buf.num_experts,
|
||||
# DeepGEMM rounds the requested capacity up for its buffer.
|
||||
num_max_tokens_per_rank=buf.num_max_tokens_per_rank,
|
||||
# Match the non-null output allocation passed to DeepGEMM.
|
||||
num_tokens=max(num_tokens, 1),
|
||||
num_topk=buf.num_topk,
|
||||
mma_type=buf.mma_type,
|
||||
)
|
||||
mega_moe_pre_dispatch(
|
||||
hidden_states,
|
||||
topk_ids_in,
|
||||
@@ -389,6 +429,10 @@ def run_mega_routed_experts(
|
||||
buf.topk_idx,
|
||||
buf.topk_weights,
|
||||
quant_group_size=32,
|
||||
shared_x_sf=(
|
||||
buf.shared_l1_acts_sf if shared_l1_weights is not None else None
|
||||
),
|
||||
shared_block_m=shared_block_m,
|
||||
)
|
||||
|
||||
# Allocate at least one row so y has a non-null CUDA data_ptr;
|
||||
@@ -404,16 +448,14 @@ def run_mega_routed_experts(
|
||||
experts.mega_l1_weights,
|
||||
experts.mega_l2_weights,
|
||||
buf,
|
||||
shared_l1_weights=shared_l1_weights,
|
||||
shared_l2_weights=shared_l2_weights,
|
||||
activation="swiglu",
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=True,
|
||||
**mega_kwargs,
|
||||
)
|
||||
y = y[:num_tokens]
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
y.mul_(routed_scaling_factor)
|
||||
return y
|
||||
return y[:num_tokens]
|
||||
|
||||
|
||||
def _interleave_mega_moe_gate_up(t: torch.Tensor, gran: int = 8) -> torch.Tensor:
|
||||
@@ -522,3 +564,36 @@ def build_mega_moe_experts_weights(experts) -> None:
|
||||
experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp)
|
||||
|
||||
experts._mega_moe_weights_built = True
|
||||
|
||||
|
||||
def build_mega_moe_shared_weights(
|
||||
shared_experts,
|
||||
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
|
||||
from deep_gemm import transform_weights_for_mega_moe
|
||||
|
||||
gate_up_proj, down_proj = shared_experts.gate_up_proj, shared_experts.down_proj
|
||||
# L1 is copied interleaved (2*moe_intermediate*hidden FP8 bytes per layer);
|
||||
# the original stays for the forward_normal fallback above the token cap.
|
||||
return transform_weights_for_mega_moe(
|
||||
(gate_up_proj.weight.data, _transform_mega_moe_shared_sf(gate_up_proj)),
|
||||
(down_proj.weight.data, _transform_mega_moe_shared_sf(down_proj)),
|
||||
)
|
||||
|
||||
|
||||
def _transform_mega_moe_shared_sf(linear) -> torch.Tensor:
|
||||
from deep_gemm import transform_sf_into_required_layout
|
||||
|
||||
from sglang.srt.layers.quantization.fp8_utils import inverse_transform_scale_ue8m0
|
||||
|
||||
n, k = linear.weight.shape
|
||||
sf = linear.weight_scale_inv.data
|
||||
if sf.dtype == torch.int32:
|
||||
# Requantized for the DeepGEMM linear runner.
|
||||
sf = inverse_transform_scale_ue8m0(sf, mn=n)[:, : k // 128]
|
||||
return transform_sf_into_required_layout(
|
||||
sf.repeat_interleave(4, dim=-1),
|
||||
mn=n,
|
||||
k=k,
|
||||
recipe=(128, 32),
|
||||
disable_ue8m0_cast=False,
|
||||
)
|
||||
|
||||
@@ -738,6 +738,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
self.shared_experts_is_fp8 = False
|
||||
self.shared_experts_weight_block_size = None
|
||||
self._shared_expert_tp1 = False
|
||||
self.mega_shared_l1_weights = None
|
||||
self.mega_shared_l2_weights = None
|
||||
# Shared experts: skip when fused into MoE kernel
|
||||
# (self.num_fused_shared_experts > 0) or when DeepEP/MegaMOE fusion is enabled.
|
||||
if (
|
||||
|
||||
@@ -5219,6 +5219,20 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self_attn.indexer.compressor.apply_ape_hotfix()
|
||||
layer.refresh_mhc_norm_weight_cache()
|
||||
|
||||
def precompile_kernels_after_loading(self) -> None:
|
||||
from sglang.srt.layers.moe.mega_moe import (
|
||||
build_mega_moe_shared_weights,
|
||||
should_fuse_mega_moe_shared_experts,
|
||||
)
|
||||
|
||||
for module in self.modules():
|
||||
if isinstance(
|
||||
module, deepseek_v2.DeepseekV2MoE
|
||||
) and should_fuse_mega_moe_shared_experts(module):
|
||||
module.mega_shared_l1_weights, module.mega_shared_l2_weights = (
|
||||
build_mega_moe_shared_weights(module.shared_experts)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def remap_weight_name_to_dpsk_hf_format(
|
||||
name: str,
|
||||
|
||||
@@ -783,6 +783,9 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
# embedding/head weights. The native CUDA path keeps the original DSpark
|
||||
# behavior and shares the target model's vocabulary modules.
|
||||
uses_own_vocab_modules = _is_npu
|
||||
precompile_kernels_after_loading = (
|
||||
DeepseekV4ForCausalLM.precompile_kernels_after_loading
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def shared_experts_fusion_disable_reason(
|
||||
|
||||
Reference in New Issue
Block a user