diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh index 03507c7d50..9bd130ee47 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh @@ -31,8 +31,11 @@ struct MegaMoEPreDispatchParams { int32_t* __restrict__ buf_x_sf; // row-major int32 [P, G/4] int64_t* __restrict__ buf_topk_idx; // [padded_max, top_k] float* __restrict__ buf_topk_weights; // [padded_max, top_k] + int32_t* __restrict__ shared_x_sf; // column-major, BLOCK_M-paged scales uint64_t buf_x_sf_stride_bytes; + uint64_t shared_x_sf_col_stride; + uint32_t shared_block_m; uint32_t num_tokens; uint32_t padded_max; uint32_t hidden; @@ -44,7 +47,7 @@ struct MegaMoEPreDispatchParams { // kMultiChunk covers rows too wide for one chunk per thread. It is a template // parameter rather than a runtime check because this kernel is issue-bound: the // loop bookkeeping alone costs ~10% on the rows that fit a single chunk. -template +template __global__ __launch_bounds__(kMaxBlockThreads, 2) void // mega_moe_pre_dispatch_kernel(const MegaMoEPreDispatchParams __grid_constant__ params) { using namespace device; @@ -66,6 +69,12 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void // const uint32_t token_id = bid; const auto token_in = params.x + static_cast(token_id) * params.hidden; const auto token_out = params.buf_x + static_cast(token_id) * params.hidden; + uint32_t shared_row = 0; + if constexpr (kWriteSharedSF) { + const uint32_t m = token_id % params.shared_block_m; + shared_row = token_id / params.shared_block_m * ((params.shared_block_m + 127u) / 128u * 128u) + m / 128u * 128u + + m % 32u * 4u + m % 128u / 32u; + } const auto quantize_chunk = [&](uint32_t chunk) { InputVec in_vec; @@ -104,6 +113,11 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void // if (within_group_id == 0 && group_id < params.num_groups) { const uint64_t byte_off = static_cast(token_id) * params.buf_x_sf_stride_bytes + group_id; reinterpret_cast(params.buf_x_sf)[byte_off] = static_cast(ue8m0_exp); + if constexpr (kWriteSharedSF) { + const uint64_t off = shared_row + static_cast(group_id / 4u) * params.shared_x_sf_col_stride; + reinterpret_cast(params.shared_x_sf)[off * sizeof(int32_t) + group_id % 4u] = + static_cast(ue8m0_exp); + } } }; @@ -144,13 +158,13 @@ __global__ __launch_bounds__(kMaxBlockThreads, 2) void // // ---- Host wrapper // ------------------------------------------------------------------------------------------------------------------------ -template +template struct MegaMoEPreDispatchKernel { static_assert(kGroupSize == 32 || kGroupSize == 64 || kGroupSize == 128, "unsupported group_size"); static constexpr auto kernel_one_chunk = - mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, false>; + mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, false, kWriteSharedSF>; static constexpr auto kernel_multi_chunk = - mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, true>; + mega_moe_pre_dispatch_kernel(kGroupSize), kUsePDL, true, kWriteSharedSF>; static void run(const tvm::ffi::TensorView x, @@ -159,7 +173,9 @@ struct MegaMoEPreDispatchKernel { const tvm::ffi::TensorView buf_x, const tvm::ffi::TensorView buf_x_sf, const tvm::ffi::TensorView buf_topk_idx, - const tvm::ffi::TensorView buf_topk_weights) { + const tvm::ffi::TensorView buf_topk_weights, + const tvm::ffi::Optional shared_x_sf, + const int64_t shared_block_m) { using namespace host; auto device = SymbolicDevice{}; @@ -232,6 +248,21 @@ struct MegaMoEPreDispatchKernel { "hidden above 8192 must be a multiple of 256 so the tail quant chunks fill whole warps"); RuntimeCheck(block_size >= top_k, "top_k must fit into one quant CTA"); + int32_t* shared_sf_ptr = nullptr; + uint64_t shared_sf_stride = 0; + if constexpr (kWriteSharedSF) { + RuntimeCheck(kGroupSize == 32, "shared MegaMoE scales require group_size=32"); + RuntimeCheck(shared_block_m > 0, "shared_block_m must be positive"); + auto S = SymbolicSize{"shared_sf_rows"}; + const auto sf = shared_x_sf.value(); + TensorMatcher({S, G4}).with_strides({1, -1}).with_dtype().with_device(device).verify(sf); + const auto rows = (num_tokens + shared_block_m - 1) / shared_block_m * ((shared_block_m + 127) / 128 * 128); + RuntimeCheck(S.unwrap() >= rows, "shared scale buffer is too small for shared_block_m"); + RuntimeCheck(sf.stride(1) >= S.unwrap(), "shared scale columns must not overlap"); + shared_sf_ptr = static_cast(sf.data_ptr()); + shared_sf_stride = sf.stride(1); + } + const auto pad_slots = (padded_max - num_tokens) * top_k; const uint32_t num_pad_blocks = pad_slots == 0 ? 0u : ((pad_slots + block_size - 1u) / block_size); const auto num_total_blocks = num_tokens + num_pad_blocks; @@ -244,7 +275,10 @@ struct MegaMoEPreDispatchKernel { .buf_x_sf = static_cast(buf_x_sf.data_ptr()), .buf_topk_idx = static_cast(buf_topk_idx.data_ptr()), .buf_topk_weights = static_cast(buf_topk_weights.data_ptr()), + .shared_x_sf = shared_sf_ptr, .buf_x_sf_stride_bytes = buf_x_sf_stride_bytes, + .shared_x_sf_col_stride = shared_sf_stride, + .shared_block_m = static_cast(shared_block_m), .num_tokens = num_tokens, .padded_max = padded_max, .hidden = hidden, diff --git a/python/sglang/kernels/ops/moe/dsv4.py b/python/sglang/kernels/ops/moe/dsv4.py index 8abfa94912..a9cf7a34f7 100644 --- a/python/sglang/kernels/ops/moe/dsv4.py +++ b/python/sglang/kernels/ops/moe/dsv4.py @@ -39,8 +39,8 @@ def _jit_hash_topk_module(): @cache_once -def _jit_mega_moe_pre_dispatch_module(quant_group_size: int): - args = make_cpp_args(quant_group_size, is_arch_support_pdl()) +def _jit_mega_moe_pre_dispatch_module(quant_group_size: int, write_shared_sf: bool): + args = make_cpp_args(quant_group_size, is_arch_support_pdl(), write_shared_sf) return load_jit( make_name("mega_moe_pre_dispatch"), *args, @@ -162,8 +162,14 @@ def mega_moe_pre_dispatch( buf_topk_idx: torch.Tensor, buf_topk_weights: torch.Tensor, quant_group_size: int = 32, + *, + shared_x_sf: Optional[torch.Tensor] = None, + shared_block_m: int = 0, ) -> None: - module = _jit_mega_moe_pre_dispatch_module(quant_group_size) + # shared_block_m must match the consuming MoE's paged scale layout + module = _jit_mega_moe_pre_dispatch_module( + quant_group_size, shared_x_sf is not None + ) module.run( x, topk_idx, @@ -172,6 +178,8 @@ def mega_moe_pre_dispatch( buf_x_sf, buf_topk_idx, buf_topk_weights, + shared_x_sf, + shared_block_m, ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 787234e08e..c2efb1299e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1288,6 +1288,7 @@ class Envs: # Blackwell MegaMoE uses a whole-grid software barrier. Keep a small # residency margin so every cluster can launch beside other streams. SGLANG_OPT_DEEPGEMM_MEGA_MOE_RESERVED_SMS = EnvInt(2) + SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS = EnvBool(True) # =================================================================== # Top-k kernels diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index cddcc1f473..324594cb10 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -34,7 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.models.deepseek_common.utils import _device_sm from sglang.srt.runtime_context import get_exec -from sglang.srt.utils import is_hip +from sglang.srt.utils import is_hip, is_sm100_supported if TYPE_CHECKING: from deep_gemm import SymmBuffer @@ -117,6 +117,7 @@ def _get_mega_moe_symm_buffer( num_topk: int, hidden: int, intermediate_hidden: int, + num_shared_experts: int = 0, mma_type: Optional[str] = None, ) -> SymmBuffer: import deep_gemm @@ -132,6 +133,7 @@ def _get_mega_moe_symm_buffer( hidden, intermediate_hidden, mma_type, + num_shared_experts, deep_gemm.get_num_sms(), ) buf = _MEGA_MOE_SYMM_BUFFER.get(key) @@ -143,6 +145,7 @@ def _get_mega_moe_symm_buffer( num_topk, hidden, intermediate_hidden, + num_shared_experts=num_shared_experts, mma_type=mma_type, activation="swiglu", ) @@ -183,6 +186,19 @@ def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool return max_tokens_per_rank <= cap +def should_fuse_mega_moe_shared_experts(moe: DeepseekV2MoE) -> bool: + # DeepGEMM's in-kernel shared expert takes FP8 weights on SM100 fp8xfp4 only. + return ( + envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_FUSE_SHARED_EXPERTS.get() + and get_moe_a2a_backend().is_megamoe() + and getattr(moe.experts, "_mega_moe_weights_built", False) + and is_sm100_supported() + and _mega_moe_mma_type(moe.experts) == "fp8xfp4" + and moe.shared_experts_is_fp8 + and moe.shared_experts_weight_block_size == [128, 128] + ) + + def forward_mega_moe( moe: DeepseekV2MoE, hidden_states: torch.Tensor, @@ -199,29 +215,29 @@ def forward_mega_moe( ) num_tokens = hidden_states.shape[0] - - sbo_overlap_flag = ( + fused_shared = moe.mega_shared_l1_weights is not None + fork_mega_moe = ( moe.alt_stream is not None - and moe.num_fused_shared_experts == 0 + and (fused_shared or moe.num_fused_shared_experts == 0) and num_tokens > 0 and get_is_capture_mode() ) - if sbo_overlap_flag: + if fork_mega_moe: current_stream = torch.cuda.current_stream() moe.alt_stream.wait_stream(current_stream) - shared_output = moe._forward_shared_experts(hidden_states) mega_stream_ctx = torch.cuda.stream(moe.alt_stream) else: - shared_output = moe._forward_shared_experts(hidden_states) mega_stream_ctx = nullcontext() + shared_output = None if fused_shared else moe._forward_shared_experts(hidden_states) + with mega_stream_ctx: y = _run_mega_routed( moe, hidden_states, forward_batch, input_ids_global, num_tokens ) - if sbo_overlap_flag: + if fork_mega_moe: current_stream.wait_stream(moe.alt_stream) if shared_output is not None: @@ -275,6 +291,11 @@ def _run_mega_routed( if moe.experts.should_fuse_routed_scaling_factor_in_topk else float(moe.routed_scaling_factor) ), + num_shared_experts=( + 0 if moe.mega_shared_l1_weights is None else moe.n_shared_experts + ), + shared_l1_weights=moe.mega_shared_l1_weights, + shared_l2_weights=moe.mega_shared_l2_weights, ) @@ -290,6 +311,9 @@ def run_mega_routed_experts( num_tokens: int, activation_clamp: Optional[float] = None, routed_scaling_factor: float = 1.0, + num_shared_experts: int = 0, + shared_l1_weights=None, + shared_l2_weights=None, ) -> torch.Tensor: # Rows are this rank's tokens; the returned rows are fully combined. import deep_gemm @@ -316,6 +340,7 @@ def run_mega_routed_experts( num_topk=top_k, hidden=hidden_size, intermediate_hidden=intermediate_size, + num_shared_experts=num_shared_experts, mma_type=mma_type, ) @@ -339,6 +364,9 @@ def run_mega_routed_experts( routed_scaling_factor=routed_scaling_factor, ) + if routed_scaling_factor != 1.0: + topk_weights_in = topk_weights_in * routed_scaling_factor + mega_kwargs = {"recipe": (1, 1, 32)} if mma_type == "nvfp4xnvfp4": # Per-token outer scales go to buf.x_scales; the kernel folds them with @@ -380,6 +408,18 @@ def run_mega_routed_experts( mma_type=mma_type, ) else: + shared_block_m = 0 + if shared_l1_weights is not None: + shared_block_m = deep_gemm.get_block_m_for_mega_moe( + num_ranks=buf.group.size(), + num_experts=buf.num_experts, + # DeepGEMM rounds the requested capacity up for its buffer. + num_max_tokens_per_rank=buf.num_max_tokens_per_rank, + # Match the non-null output allocation passed to DeepGEMM. + num_tokens=max(num_tokens, 1), + num_topk=buf.num_topk, + mma_type=buf.mma_type, + ) mega_moe_pre_dispatch( hidden_states, topk_ids_in, @@ -389,6 +429,10 @@ def run_mega_routed_experts( buf.topk_idx, buf.topk_weights, quant_group_size=32, + shared_x_sf=( + buf.shared_l1_acts_sf if shared_l1_weights is not None else None + ), + shared_block_m=shared_block_m, ) # Allocate at least one row so y has a non-null CUDA data_ptr; @@ -404,16 +448,14 @@ def run_mega_routed_experts( experts.mega_l1_weights, experts.mega_l2_weights, buf, + shared_l1_weights=shared_l1_weights, + shared_l2_weights=shared_l2_weights, activation="swiglu", activation_clamp=activation_clamp, fast_math=True, **mega_kwargs, ) - y = y[:num_tokens] - - if routed_scaling_factor != 1.0: - y.mul_(routed_scaling_factor) - return y + return y[:num_tokens] def _interleave_mega_moe_gate_up(t: torch.Tensor, gran: int = 8) -> torch.Tensor: @@ -522,3 +564,36 @@ def build_mega_moe_experts_weights(experts) -> None: experts.mega_l2_weights = (experts.w2_weight.data, w2_sf_utccp) experts._mega_moe_weights_built = True + + +def build_mega_moe_shared_weights( + shared_experts, +) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]: + from deep_gemm import transform_weights_for_mega_moe + + gate_up_proj, down_proj = shared_experts.gate_up_proj, shared_experts.down_proj + # L1 is copied interleaved (2*moe_intermediate*hidden FP8 bytes per layer); + # the original stays for the forward_normal fallback above the token cap. + return transform_weights_for_mega_moe( + (gate_up_proj.weight.data, _transform_mega_moe_shared_sf(gate_up_proj)), + (down_proj.weight.data, _transform_mega_moe_shared_sf(down_proj)), + ) + + +def _transform_mega_moe_shared_sf(linear) -> torch.Tensor: + from deep_gemm import transform_sf_into_required_layout + + from sglang.srt.layers.quantization.fp8_utils import inverse_transform_scale_ue8m0 + + n, k = linear.weight.shape + sf = linear.weight_scale_inv.data + if sf.dtype == torch.int32: + # Requantized for the DeepGEMM linear runner. + sf = inverse_transform_scale_ue8m0(sf, mn=n)[:, : k // 128] + return transform_sf_into_required_layout( + sf.repeat_interleave(4, dim=-1), + mn=n, + k=k, + recipe=(128, 32), + disable_ue8m0_cast=False, + ) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 0181eb7b91..6ce92987a5 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -738,6 +738,8 @@ class DeepseekV2MoE(nn.Module): self.shared_experts_is_fp8 = False self.shared_experts_weight_block_size = None self._shared_expert_tp1 = False + self.mega_shared_l1_weights = None + self.mega_shared_l2_weights = None # Shared experts: skip when fused into MoE kernel # (self.num_fused_shared_experts > 0) or when DeepEP/MegaMOE fusion is enabled. if ( diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index c28eb599fe..b6b31eb5cb 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -5219,6 +5219,20 @@ class DeepseekV4ForCausalLM(nn.Module): self_attn.indexer.compressor.apply_ape_hotfix() layer.refresh_mhc_norm_weight_cache() + def precompile_kernels_after_loading(self) -> None: + from sglang.srt.layers.moe.mega_moe import ( + build_mega_moe_shared_weights, + should_fuse_mega_moe_shared_experts, + ) + + for module in self.modules(): + if isinstance( + module, deepseek_v2.DeepseekV2MoE + ) and should_fuse_mega_moe_shared_experts(module): + module.mega_shared_l1_weights, module.mega_shared_l2_weights = ( + build_mega_moe_shared_weights(module.shared_experts) + ) + @staticmethod def remap_weight_name_to_dpsk_hf_format( name: str, diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index baebc2de6f..30e2f205c3 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -783,6 +783,9 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): # embedding/head weights. The native CUDA path keeps the original DSpark # behavior and shares the target model's vocabulary modules. uses_own_vocab_modules = _is_npu + precompile_kernels_after_loading = ( + DeepseekV4ForCausalLM.precompile_kernels_after_loading + ) @classmethod def shared_experts_fusion_disable_reason( diff --git a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py index 66b219e5d6..8a706bf962 100644 --- a/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py +++ b/test/registered/unit/layers/moe/test_mega_moe_deepgemm_api.py @@ -240,6 +240,8 @@ class TestDeepGemmMegaMoeApi(CustomTestCase): num_fused_shared_experts=0, layer_id=0, routed_scaling_factor=1.0, + mega_shared_l1_weights=None, + mega_shared_l2_weights=None, ) with (