[FlashInfer] Add startup autotuning for MegaMoE (#43789)

This commit is contained in:
Ziang Li
2026-10-13 02:30:31 -07:00
committed by GitHub
parent 195cb7afb4
commit 01f6840693
9 changed files with 879 additions and 85 deletions
+4 -3
View File
@@ -1097,9 +1097,10 @@ class Envs:
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
# cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging
# region and can win at large batch, but makes the output accumulation order
# nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility.
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
# FlashInfer's tuning profile may still select external reduction.
# BF16 atomic accumulation is nondeterministic -- keep off for
# bit-reproducibility.
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False)
# Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE
@@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor:
def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]:
import inspect
if "return_workspace_view" in inspect.signature(mega.forward).parameters:
if mega.supports_output_view:
return _forward_megamoe_with_workspace_view
return _forward_megamoe_legacy
@@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer(
layer.hidden_size,
)
process_group = None
if world_size > 1:
from sglang.srt.runtime_context import get_parallel
process_group = get_parallel().moe_ep_group.device_group
bootstrap = BootstrapConfig(
world_size=world_size,
rank=rank,
device=torch.cuda.current_device(),
process_group=process_group,
)
fleet_params = FleetParams(
num_experts=layer.num_experts,
max_tokens_per_rank=max_tokens_per_rank,
token_hidden_size=layer.hidden_size,
)
backend = MegaConfig(
megakernel=megakernel_config,
preprocess_weights=False,
transformed_weights=transformed_weights,
)
mega = MoEEpMegaLayer(
bootstrap=BootstrapConfig(
world_size=world_size, rank=rank, device=torch.cuda.current_device()
),
fleet_params=FleetParams(
num_experts=layer.num_experts,
max_tokens_per_rank=max_tokens_per_rank,
token_hidden_size=layer.hidden_size,
),
bootstrap=bootstrap,
fleet_params=fleet_params,
# weights already preprocessed in prepare_*; with transformed_weights set
# the kernel never reads `weights` (see MoEEpMegaLayer), so pass None.
weights=None,
backend=MegaConfig(
megakernel=megakernel_config,
preprocess_weights=False,
transformed_weights=transformed_weights,
),
backend=backend,
)
layer._flashinfer_megamoe_layer = mega
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
if hasattr(megakernel_config, "knobs"):
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
MegaMoeTunedForward,
)
layer._flashinfer_megamoe_forward = MegaMoeTunedForward(
bootstrap, fleet_params, backend
)
else:
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
return mega
@@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
apply_topk_in_fc1=True,
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
input_norm_const=input_norm_const,
fc1_alpha=layer.g1_alphas,
fc2_alpha=layer.g2_alphas,
fc1_norm_const=layer.w2_input_scale_quant,
),
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
@@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
kind="mxfp8_e4m3",
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
),
w13_scale=layer.w13_weight_scale_inv,
w2_scale=layer.w2_weight_scale_inv,
@@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
)
def _ensure_shared_workspace(mega: Any) -> None:
"""Share this layer's workspace across MegaMOE layers with identical
fleet/kernel geometry.
FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const
tensor identity, but sglang binds distinct tensor objects per layer, so
its pool never hits across layers; key on geometry instead, since those
values are re-staged into the workspace per forward rather than baked
into the compiled kernel. Creation is collective, so it must only happen
on a layer's first forward, under warmup's cross-rank lockstep.
"""
if mega._workspace is not None:
return
fp = mega._fleet_params
kc = mega._megakernel_config
mc = mega._mega_config
from sglang.srt.runtime_context import get_resources
key = (
getattr(kc, "kernel_name", kc.__class__.__name__),
mega._bootstrap.world_size,
fp.num_experts,
fp.max_tokens_per_rank,
fp.token_hidden_size,
kc.top_k,
kc.intermediate_size,
getattr(kc, "gate_up_clamp", None),
getattr(kc, "activation_clamp", None),
getattr(kc, "apply_topk_in_fc1", None),
getattr(kc, "kind", None),
getattr(kc, "in_kernel_fc2_reduce", None),
getattr(kc, "combine_dtype", None),
getattr(kc, "token_back_by_dispatch", None),
getattr(kc, "fast_math", None),
mc.quantize_input,
)
workspaces = get_resources().flashinfer_megamoe_workspaces
shared = workspaces.get(key)
if shared is None:
workspaces[key] = mega._ensure_workspace()
else:
mega._workspace = shared
@register_fused_func("flashinfer_megamoe", "flashinfer_megamoe")
def run_flashinfer_megamoe(
dispatch_output: DispatchOutput,
@@ -599,7 +571,6 @@ def run_flashinfer_megamoe(
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
mega = quant_info.mega
_ensure_shared_workspace(mega)
t = MoEEpTensors(
hidden_states=x.to(torch.bfloat16),
@@ -607,6 +578,7 @@ def run_flashinfer_megamoe(
# directly into its final int64 workspace buffer. Keep this path copy-free.
topk_ids=topk_ids,
topk_weights=topk_weights.to(torch.float32),
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
fc1_alpha=quant_info.fc1_alpha,
fc2_alpha=quant_info.fc2_alpha,
fc1_norm_const=quant_info.fc1_norm_const,
@@ -0,0 +1,366 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE."""
from __future__ import annotations
import json
import logging
import os
from bisect import bisect_left
from contextlib import contextmanager
from dataclasses import dataclass, field, fields, replace
from functools import cached_property
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
import torch
logger = logging.getLogger(__name__)
@dataclass
class _AutotuneContext:
decode_num_tokens: int
prefill_num_tokens: int
directory: Path
records: dict
staged_profiles: dict[str, Path] = field(default_factory=dict)
dirty: bool = False
_active_context: _AutotuneContext | None = None
# Native workspace pooling spans startup contexts. Keep their
# selected tactics consistent too; a cache-path change must not retune storage
# already captured by another layer. These payloads own no weights or tensors.
_session_winners: dict[tuple[Any, int, str], dict] = {}
_CACHE_NAMESPACE = "sglang_flashinfer_megamoe"
@contextmanager
def megamoe_autotune_context(
decode_num_tokens: int,
prefill_num_tokens: int,
):
"""Prepare profiles inside the engine's pre-capture FlashInfer autotune pass."""
from flashinfer.autotuner import AutoTuner
global _active_context
tuner = AutoTuner.get()
previous = _active_context
with TemporaryDirectory(prefix="sglang-megamoe-") as directory:
context = _AutotuneContext(
decode_num_tokens,
prefill_num_tokens,
Path(directory),
tuner.get_namespaced_records(_CACHE_NAMESPACE),
)
_active_context = context
try:
yield
finally:
_active_context = previous
if context.dirty:
tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records)
@contextmanager
def _native_cache(path: Path):
# Native AUTO exposes its winner through this file; generic autotune owns
# persistence. Each geometry is staged once per startup context.
name = "FLASHINFER_MOE_EP_KNOB_CACHE"
previous = os.environ.get(name)
os.environ[name] = str(path)
try:
yield
finally:
if previous is None:
os.environ.pop(name, None)
else:
os.environ[name] = previous
def _capacities(limit: int) -> list[int]:
limit = max(1, limit)
return sorted({1 << i for i in range(limit.bit_length())} | {limit})
def _profile_inputs(tensors, capacity: int, num_experts: int):
count = tensors.num_tokens
if count:
def resize(tensor):
if capacity <= count:
return tensor[:capacity]
return tensor.repeat(
((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1)
)[:capacity]
hidden = resize(tensors.hidden_states)
ids = resize(tensors.topk_ids)
scores = resize(tensors.topk_weights)
scales = resize(tensors.scales) if tensors.scales is not None else None
else:
# Idle ranks still enter every collective tuning iteration.
hidden = tensors.hidden_states.new_zeros(
(capacity, tensors.hidden_states.shape[1])
)
top_k = tensors.topk_ids.shape[1]
ids = (
torch.arange(
capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device
).reshape(capacity, top_k)
% num_experts
)
scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k)
scales = (
tensors.scales.new_ones((capacity, *tensors.scales.shape[1:]))
if tensors.scales is not None
else None
)
return replace(
tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales
)
def _valid_native_cache(payload) -> bool:
if not isinstance(payload, dict) or payload.get("version") != 1:
return False
entries = payload.get("entries")
return (
isinstance(entries, list)
and len(entries) == 1
and isinstance(entries[0], dict)
and isinstance(entries[0].get("knobs"), dict)
and bool(entries[0]["knobs"])
)
class MegaMoeTunedForward:
"""Own public workspace handles; FlashInfer pools their storage across layers.
The caller installs this for backends with prepared weights and default
``knobs=None``. Preparation must precede ordinary forwarding and capture.
"""
def __init__(self, bootstrap, fleet_params, backend):
if backend.megakernel.knobs is not None:
raise ValueError("MegaMoE startup tuning requires knobs=None")
if backend.preprocess_weights or backend.transformed_weights is None:
raise ValueError("MegaMoE startup tuning requires prepared weights")
self.bootstrap = bootstrap
self.fleet_params = fleet_params
self.backend = backend
self.workspaces: dict[int, Any] = {}
self._fallback_used = False
self._profile_capacities: tuple[int, ...] = ()
def _common_tokens(self, tensors) -> int:
if self.bootstrap.world_size == 1:
return tensors.num_tokens
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
counts = get_dp_global_num_tokens()
if counts is None:
raise RuntimeError(
"MegaMoE workspace selection requires synchronized DP token counts"
)
return max(counts, default=0)
@cached_property
def _geometry_key(self) -> str:
config = {}
for item in fields(self.backend.megakernel):
# Per-layer scale values do not change the pooled kernel geometry.
if item.name in (
"knobs",
"input_norm_const",
"fc1_norm_const",
"fc1_alpha",
"fc2_alpha",
):
continue
value = getattr(self.backend.megakernel, item.name)
if not isinstance(value, torch.Tensor):
config[item.name] = value
# The enclosing flashinfer.autotune context validates runtime/compiler
# metadata. This key only adds the MegaMoE configuration and geometry.
return json.dumps(
{
"torch": torch.__version__,
"mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"),
"config": config,
"world_size": self.bootstrap.world_size,
"hidden": self.fleet_params.token_hidden_size,
"num_experts": self.fleet_params.num_experts,
},
sort_keys=True,
)
def _require_all_ranks(self, success: bool, message: str) -> None:
if self.bootstrap.world_size > 1:
import torch.distributed as dist
ready = torch.tensor(int(success), dtype=torch.int32, device="cuda")
dist.all_reduce(
ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group
)
success = bool(ready.item())
if not success:
raise RuntimeError(message)
def _broadcast(self, value):
if self.bootstrap.world_size == 1:
return value
import torch.distributed as dist
group = self.bootstrap.process_group
values = [value]
dist.broadcast_object_list(values, group=group, group_src=0)
return values[0]
def _tune(self, tensors, capacity: int, native_path: Path):
from flashinfer.moe_ep import MoEEpMegaLayer
backend = replace(
self.backend, megakernel=replace(self.backend.megakernel, knobs="auto")
)
temporary = MoEEpMegaLayer(
bootstrap=self.bootstrap,
fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity),
weights=None,
backend=backend,
)
payload = None
try:
temporary.warmup(tensors)
if self.bootstrap.rank == 0:
try:
payload = json.loads(native_path.read_text())
except (OSError, ValueError):
pass
finally:
temporary.destroy()
payload = self._broadcast(payload)
if not _valid_native_cache(payload):
raise RuntimeError("MegaMoE autotune did not record its selected tactic")
return payload
def _prepare_profile(self, mega, tensors, capacity: int) -> None:
context = _active_context
assert context is not None
profile_key = f"{self._geometry_key}:{capacity}"
profile_inputs = _profile_inputs(
tensors, capacity, self.fleet_params.num_experts
)
native_path = context.staged_profiles.get(profile_key)
needs_staging = native_path is None
if needs_staging:
native_path = (
context.directory / f"profile{len(context.staged_profiles)}.json"
)
with _native_cache(native_path):
if needs_staging:
key = (
self.bootstrap.process_group,
torch.cuda.current_device(),
profile_key,
)
payload = _session_winners.get(key)
if payload is None:
payload = self._broadcast(
context.records.get(profile_key)
if self.bootstrap.rank == 0
else None
)
source = "cache hit"
if not _valid_native_cache(payload):
payload = self._tune(profile_inputs, capacity, native_path)
source = "tuned"
_session_winners[key] = payload
if self.bootstrap.rank == 0:
logger.info(
"MegaMoE %s: capacity=%s, knobs=%s",
source,
capacity,
payload["entries"][0]["knobs"],
)
# The backend owns dtype/layout keys and loads its original payload.
try:
native_path.write_text(json.dumps(payload))
written = True
except OSError:
written = False
self._require_all_ranks(
written,
"Cannot stage MegaMoE's selected tactic for workspace creation",
)
context.staged_profiles[profile_key] = native_path
if context.records.get(profile_key) != payload:
context.records[profile_key] = payload
context.dirty = True
workspace = mega.create_workspace(capacity)
mega.warmup(profile_inputs, workspace=workspace)
self.workspaces[capacity] = workspace
@staticmethod
def _forward(mega, tensors, workspace=None):
if mega.supports_output_view:
return mega.forward(
tensors, workspace=workspace, return_workspace_view=True
)
return mega.forward(tensors, workspace=workspace)
def __call__(self, mega, tensors):
context = _active_context
if context is None and not self.workspaces:
self._fallback_used = True
return self._forward(mega, tensors)
if context is not None and not self.workspaces:
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"MegaMoE autotune must complete before CUDA graph capture"
)
if self._fallback_used:
raise RuntimeError(
"MegaMoE autotune must precede its first ordinary forward"
)
import torch.distributed as dist
if (
dist.is_initialized()
and self.bootstrap.world_size != dist.get_world_size()
):
raise RuntimeError(
"FlashInfer MegaMoE startup tuning currently requires the full WORLD process group"
)
# The startup dummy can be much larger than captured decode batches.
# Keep small-N profiles bounded by decode, plus one prefill profile.
capacities = set(_capacities(context.decode_num_tokens))
capacities.add(context.prefill_num_tokens)
for capacity in sorted(capacities):
self._prepare_profile(mega, tensors, capacity)
self._profile_capacities = tuple(sorted(self.workspaces))
count = self._common_tokens(tensors)
index = bisect_left(self._profile_capacities, count)
if index < len(self._profile_capacities):
return self._forward(
mega, tensors, self.workspaces[self._profile_capacities[index]]
)
raise ValueError(
f"MegaMoE received {count} tokens per rank, exceeding its prepared "
f"decode/prefill capacity {max(self.workspaces)}"
)
@@ -19,6 +19,7 @@ import functools
import hashlib
import json
import logging
import uuid
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Optional
@@ -34,8 +35,10 @@ from sglang.srt.runtime_context import (
get_schedule,
get_spec,
max_prefill_buffer_tokens,
max_speculative_num_draft_tokens,
)
from sglang.srt.utils import empty_context, log_info_on_rank0
from sglang.srt.utils.common import get_eager_max_batch_size
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
@@ -94,6 +97,7 @@ def should_run_flashinfer_autotune(
"flashinfer_mxfp4",
"flashinfer_cutedsl",
"flashinfer_cutlass",
"flashinfer_megamoe",
]
from sglang.srt.layers.quantization.fp4_utils import (
@@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path:
/ cache_key
)
cache_dir.mkdir(parents=True, exist_ok=True)
return (
cache_dir
/ f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json"
)
rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}"
if (
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
and not mr.is_draft_worker
):
from sglang.srt.platforms import current_platform
# Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's
# namespaced records currently require one writer per file. Follow the
# P2P cache's device isolation, using physical UUIDs for stable identity
# across CUDA index remapping and restarts. Engines sharing physical GPUs
# must use separate SGLANG_CACHE_DIRs.
rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}"
return cache_dir / f"{rank_key}.json"
def _autotune_tactic_sync_group(
@@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
runs_dir = cache_path.parent / "runs"
runs_dir.mkdir(parents=True, exist_ok=True)
autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}"
autotune_cache = runs_dir / (
f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}"
)
logger.info(
"Running FlashInfer autotune (cache reuse DISABLED via "
"SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s",
@@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
mr.forward_stream.wait_stream(torch.cuda.current_stream())
with torch.get_device_module(mr.device).stream(mr.forward_stream):
from sglang.srt.layers.logits_processor import autotune_dummy_run_mode
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
megamoe_autotune_context,
)
skip_ops = get_flashinfer_autotune_skip_ops(mr)
# autotune(cache=...) clears all file-loaded tactics on entry, which would drop
@@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
tuner = AutoTuner.get()
if reuse_cache and autotune_cache.is_file():
tuner.load_configs(str(autotune_cache))
mega_context = empty_context()
if (
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
and not mr.is_draft_worker
and "flashinfer_megamoe" not in skip_ops
):
# The chunk size is already per DP rank. Prepare one prefill profile
# independently of the optional full-model EXTEND autotune pass.
prefill_num_tokens = (
max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
)
mega_context = megamoe_autotune_context(
decode_num_tokens=max(
mr.max_decode_logits_rows(),
get_eager_max_batch_size(mr.max_running_requests)
* mr.decode_num_tokens_per_req(
num_draft_tokens=max_speculative_num_draft_tokens()
),
),
prefill_num_tokens=prefill_num_tokens,
)
with (
_autotune_process_group(sync_group),
autotune(
@@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
cache=None if reuse_cache else str(autotune_cache),
skip_ops=skip_ops,
),
mega_context,
autotune_dummy_run_mode(run_lm_head=run_lm_head),
):
yield
-1
View File
@@ -644,7 +644,6 @@ class Resources(_FlagGroupBase):
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
tbo_event_pool: dict = msgspec.field(default_factory=dict)
flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
# State capturers (installed by their subsystems when capture is on).
indexer_capturer: Any = None
experts_capturer: Any = None
+5
View File
@@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
elif moe_a2a_backend.is_flashinfer_megamoe():
# MegaMoE's capacity profiles require the same decode graph
# bucket on every EP rank. As for FlashInfer A2A above, this only
# enables DP-count bookkeeping; expert inputs remain local.
return True
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
# Same bookkeeping, for the same reason. Bounding mori's receive
# buffer means baking a fan-in size into a captured graph, and the