[Perf] [KDA] muse_spark: use BLOCK_M 240 for BF16 MegaMoE prefill shards (#45637)

Co-authored-by: kernel-design-agents <kernel-design-agents@users.noreply.github.com>
This commit is contained in:
Song Bian
2026-10-22 20:57:30 +08:00
committed by GitHub
co-authored by kernel-design-agents
parent a9902f47e3
commit 635836129b
@@ -8,7 +8,10 @@ per-expert norm before the route-weighted sum, using DeepGEMM.
from __future__ import annotations
import contextlib
import functools
import logging
import os
import torch
@@ -16,6 +19,13 @@ logger = logging.getLogger(__name__)
_MEGAMOE_SYMM_BUFFERS: dict = {}
# DeepGEMM reads this on every MegaMoE launch; an empty value keeps its heuristic.
_TILES_ENV = "DG_MEGA_MOE_TILES"
# (CTA pair, BLOCK_M, STORE_BLOCK_M, BLOCK_K bytes, epilogue warpgroups). With
# Muse routing on GB300, BLOCK_M 240 beats the heuristic's 192 for BF16
# prefill-sized shards; smaller heuristic tiles are kept.
_BF16_PREFILL_TILES = "2,240,40,128,2"
def _dimensions(layer):
if hasattr(layer, "global_num_experts"):
@@ -306,6 +316,49 @@ def _get_symm_buffer(layer, group, capacity, device):
return buf
@functools.lru_cache(maxsize=4096)
def _heuristic_block_m(num_ranks, num_experts, capacity, num_tokens, top_k, mma_type):
import deep_gemm
return deep_gemm.get_block_m_for_mega_moe(
num_ranks=num_ranks,
num_experts=num_experts,
num_max_tokens_per_rank=capacity,
num_tokens=num_tokens,
num_topk=top_k,
mma_type=mma_type,
)
def _tile_override(buf, num_tokens):
"""BF16 prefill tiles for this launch, or None to keep DeepGEMM's choice."""
if buf.mma_type != "bf16xbf16" or os.environ.get(_TILES_ENV):
# A user-provided DG_MEGA_MOE_TILES always wins.
return None
block_m = _heuristic_block_m(
buf.group.size(),
buf.num_experts,
buf.num_max_tokens_per_rank,
num_tokens,
buf.num_topk,
buf.mma_type,
)
return _BF16_PREFILL_TILES if block_m == 192 else None
@contextlib.contextmanager
def _megamoe_tiles(tiles):
if tiles is None:
yield
return
os.environ[_TILES_ENV] = tiles
try:
yield
finally:
# Keep the key (empty = heuristic) so later launches only replace values.
os.environ[_TILES_ENV] = ""
@torch.no_grad()
def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_norm):
"""Dispatch sequence shards, run both GEMMs, normalize experts, then combine.
@@ -398,5 +451,7 @@ def forward_megamoe(layer, hidden_states, topk_scores, topk_ids, post_expert_nor
kernel = deep_gemm.nvfp4_mega_moe
else:
kernel = deep_gemm.bf16_mega_moe
kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs)
# Match the non-null output allocation passed to DeepGEMM.
with _megamoe_tiles(_tile_override(buf, y.shape[0])):
kernel(y, layer.megamoe_l1_weights, layer.megamoe_l2_weights, buf, **kwargs)
return y[:n]