[FlashInfer] Add startup autotuning for MegaMoE (#43789)

This commit is contained in:
Ziang Li
2026-10-13 02:30:31 -07:00
committed by GitHub
parent 195cb7afb4
commit 01f6840693
9 changed files with 879 additions and 85 deletions
@@ -164,7 +164,7 @@ SGLang supports various environment variables that can be used to configure its
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`SGLANG_FLASHINFER_AUTOTUNE_CACHE`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs. Set to `0` to force re-autotuning on every startup; the fresh result is written to a `runs/<rank>.<timestamp>.json` sibling file (the canonical cache is left untouched).</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs, including MegaMoE capacity profiles. Set to `0` to disable disk-cache reuse; the fresh result is written to a `runs/<rank>.<timestamp>.<uuid>.json` sibling file (the canonical cache is left untouched). Target MegaMoE rank files include the physical GPU UUID. Use separate `SGLANG_CACHE_DIR` roots for engines sharing physical GPUs; a running process retains tactics for already-pooled workspaces.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`true`</td>
</tr>
<tr>
@@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the CuTe DSL MegaMOE in-kernel FC2 reduction. This can reduce workspace size and improve large-batch performance, but BF16 atomic accumulation is nondeterministic. It is incompatible with quantized combine dtypes.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine.</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>"false"</code></td>
</tr>
<tr>
+4 -3
View File
@@ -1097,9 +1097,10 @@ class Envs:
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
# cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging
# region and can win at large batch, but makes the output accumulation order
# nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility.
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
# FlashInfer's tuning profile may still select external reduction.
# BF16 atomic accumulation is nondeterministic -- keep off for
# bit-reproducibility.
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False)
# Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE
@@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor:
def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]:
import inspect
if "return_workspace_view" in inspect.signature(mega.forward).parameters:
if mega.supports_output_view:
return _forward_megamoe_with_workspace_view
return _forward_megamoe_legacy
@@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer(
layer.hidden_size,
)
process_group = None
if world_size > 1:
from sglang.srt.runtime_context import get_parallel
process_group = get_parallel().moe_ep_group.device_group
bootstrap = BootstrapConfig(
world_size=world_size,
rank=rank,
device=torch.cuda.current_device(),
process_group=process_group,
)
fleet_params = FleetParams(
num_experts=layer.num_experts,
max_tokens_per_rank=max_tokens_per_rank,
token_hidden_size=layer.hidden_size,
)
backend = MegaConfig(
megakernel=megakernel_config,
preprocess_weights=False,
transformed_weights=transformed_weights,
)
mega = MoEEpMegaLayer(
bootstrap=BootstrapConfig(
world_size=world_size, rank=rank, device=torch.cuda.current_device()
),
fleet_params=FleetParams(
num_experts=layer.num_experts,
max_tokens_per_rank=max_tokens_per_rank,
token_hidden_size=layer.hidden_size,
),
bootstrap=bootstrap,
fleet_params=fleet_params,
# weights already preprocessed in prepare_*; with transformed_weights set
# the kernel never reads `weights` (see MoEEpMegaLayer), so pass None.
weights=None,
backend=MegaConfig(
megakernel=megakernel_config,
preprocess_weights=False,
transformed_weights=transformed_weights,
),
backend=backend,
)
layer._flashinfer_megamoe_layer = mega
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
if hasattr(megakernel_config, "knobs"):
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
MegaMoeTunedForward,
)
layer._flashinfer_megamoe_forward = MegaMoeTunedForward(
bootstrap, fleet_params, backend
)
else:
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
return mega
@@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
apply_topk_in_fc1=True,
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
input_norm_const=input_norm_const,
fc1_alpha=layer.g1_alphas,
fc2_alpha=layer.g2_alphas,
fc1_norm_const=layer.w2_input_scale_quant,
),
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
@@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
kind="mxfp8_e4m3",
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
),
w13_scale=layer.w13_weight_scale_inv,
w2_scale=layer.w2_weight_scale_inv,
@@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
)
def _ensure_shared_workspace(mega: Any) -> None:
"""Share this layer's workspace across MegaMOE layers with identical
fleet/kernel geometry.
FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const
tensor identity, but sglang binds distinct tensor objects per layer, so
its pool never hits across layers; key on geometry instead, since those
values are re-staged into the workspace per forward rather than baked
into the compiled kernel. Creation is collective, so it must only happen
on a layer's first forward, under warmup's cross-rank lockstep.
"""
if mega._workspace is not None:
return
fp = mega._fleet_params
kc = mega._megakernel_config
mc = mega._mega_config
from sglang.srt.runtime_context import get_resources
key = (
getattr(kc, "kernel_name", kc.__class__.__name__),
mega._bootstrap.world_size,
fp.num_experts,
fp.max_tokens_per_rank,
fp.token_hidden_size,
kc.top_k,
kc.intermediate_size,
getattr(kc, "gate_up_clamp", None),
getattr(kc, "activation_clamp", None),
getattr(kc, "apply_topk_in_fc1", None),
getattr(kc, "kind", None),
getattr(kc, "in_kernel_fc2_reduce", None),
getattr(kc, "combine_dtype", None),
getattr(kc, "token_back_by_dispatch", None),
getattr(kc, "fast_math", None),
mc.quantize_input,
)
workspaces = get_resources().flashinfer_megamoe_workspaces
shared = workspaces.get(key)
if shared is None:
workspaces[key] = mega._ensure_workspace()
else:
mega._workspace = shared
@register_fused_func("flashinfer_megamoe", "flashinfer_megamoe")
def run_flashinfer_megamoe(
dispatch_output: DispatchOutput,
@@ -599,7 +571,6 @@ def run_flashinfer_megamoe(
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
mega = quant_info.mega
_ensure_shared_workspace(mega)
t = MoEEpTensors(
hidden_states=x.to(torch.bfloat16),
@@ -607,6 +578,7 @@ def run_flashinfer_megamoe(
# directly into its final int64 workspace buffer. Keep this path copy-free.
topk_ids=topk_ids,
topk_weights=topk_weights.to(torch.float32),
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
fc1_alpha=quant_info.fc1_alpha,
fc2_alpha=quant_info.fc2_alpha,
fc1_norm_const=quant_info.fc1_norm_const,
@@ -0,0 +1,366 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE."""
from __future__ import annotations
import json
import logging
import os
from bisect import bisect_left
from contextlib import contextmanager
from dataclasses import dataclass, field, fields, replace
from functools import cached_property
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
import torch
logger = logging.getLogger(__name__)
@dataclass
class _AutotuneContext:
decode_num_tokens: int
prefill_num_tokens: int
directory: Path
records: dict
staged_profiles: dict[str, Path] = field(default_factory=dict)
dirty: bool = False
_active_context: _AutotuneContext | None = None
# Native workspace pooling spans startup contexts. Keep their
# selected tactics consistent too; a cache-path change must not retune storage
# already captured by another layer. These payloads own no weights or tensors.
_session_winners: dict[tuple[Any, int, str], dict] = {}
_CACHE_NAMESPACE = "sglang_flashinfer_megamoe"
@contextmanager
def megamoe_autotune_context(
decode_num_tokens: int,
prefill_num_tokens: int,
):
"""Prepare profiles inside the engine's pre-capture FlashInfer autotune pass."""
from flashinfer.autotuner import AutoTuner
global _active_context
tuner = AutoTuner.get()
previous = _active_context
with TemporaryDirectory(prefix="sglang-megamoe-") as directory:
context = _AutotuneContext(
decode_num_tokens,
prefill_num_tokens,
Path(directory),
tuner.get_namespaced_records(_CACHE_NAMESPACE),
)
_active_context = context
try:
yield
finally:
_active_context = previous
if context.dirty:
tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records)
@contextmanager
def _native_cache(path: Path):
# Native AUTO exposes its winner through this file; generic autotune owns
# persistence. Each geometry is staged once per startup context.
name = "FLASHINFER_MOE_EP_KNOB_CACHE"
previous = os.environ.get(name)
os.environ[name] = str(path)
try:
yield
finally:
if previous is None:
os.environ.pop(name, None)
else:
os.environ[name] = previous
def _capacities(limit: int) -> list[int]:
limit = max(1, limit)
return sorted({1 << i for i in range(limit.bit_length())} | {limit})
def _profile_inputs(tensors, capacity: int, num_experts: int):
count = tensors.num_tokens
if count:
def resize(tensor):
if capacity <= count:
return tensor[:capacity]
return tensor.repeat(
((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1)
)[:capacity]
hidden = resize(tensors.hidden_states)
ids = resize(tensors.topk_ids)
scores = resize(tensors.topk_weights)
scales = resize(tensors.scales) if tensors.scales is not None else None
else:
# Idle ranks still enter every collective tuning iteration.
hidden = tensors.hidden_states.new_zeros(
(capacity, tensors.hidden_states.shape[1])
)
top_k = tensors.topk_ids.shape[1]
ids = (
torch.arange(
capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device
).reshape(capacity, top_k)
% num_experts
)
scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k)
scales = (
tensors.scales.new_ones((capacity, *tensors.scales.shape[1:]))
if tensors.scales is not None
else None
)
return replace(
tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales
)
def _valid_native_cache(payload) -> bool:
if not isinstance(payload, dict) or payload.get("version") != 1:
return False
entries = payload.get("entries")
return (
isinstance(entries, list)
and len(entries) == 1
and isinstance(entries[0], dict)
and isinstance(entries[0].get("knobs"), dict)
and bool(entries[0]["knobs"])
)
class MegaMoeTunedForward:
"""Own public workspace handles; FlashInfer pools their storage across layers.
The caller installs this for backends with prepared weights and default
``knobs=None``. Preparation must precede ordinary forwarding and capture.
"""
def __init__(self, bootstrap, fleet_params, backend):
if backend.megakernel.knobs is not None:
raise ValueError("MegaMoE startup tuning requires knobs=None")
if backend.preprocess_weights or backend.transformed_weights is None:
raise ValueError("MegaMoE startup tuning requires prepared weights")
self.bootstrap = bootstrap
self.fleet_params = fleet_params
self.backend = backend
self.workspaces: dict[int, Any] = {}
self._fallback_used = False
self._profile_capacities: tuple[int, ...] = ()
def _common_tokens(self, tensors) -> int:
if self.bootstrap.world_size == 1:
return tensors.num_tokens
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
counts = get_dp_global_num_tokens()
if counts is None:
raise RuntimeError(
"MegaMoE workspace selection requires synchronized DP token counts"
)
return max(counts, default=0)
@cached_property
def _geometry_key(self) -> str:
config = {}
for item in fields(self.backend.megakernel):
# Per-layer scale values do not change the pooled kernel geometry.
if item.name in (
"knobs",
"input_norm_const",
"fc1_norm_const",
"fc1_alpha",
"fc2_alpha",
):
continue
value = getattr(self.backend.megakernel, item.name)
if not isinstance(value, torch.Tensor):
config[item.name] = value
# The enclosing flashinfer.autotune context validates runtime/compiler
# metadata. This key only adds the MegaMoE configuration and geometry.
return json.dumps(
{
"torch": torch.__version__,
"mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"),
"config": config,
"world_size": self.bootstrap.world_size,
"hidden": self.fleet_params.token_hidden_size,
"num_experts": self.fleet_params.num_experts,
},
sort_keys=True,
)
def _require_all_ranks(self, success: bool, message: str) -> None:
if self.bootstrap.world_size > 1:
import torch.distributed as dist
ready = torch.tensor(int(success), dtype=torch.int32, device="cuda")
dist.all_reduce(
ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group
)
success = bool(ready.item())
if not success:
raise RuntimeError(message)
def _broadcast(self, value):
if self.bootstrap.world_size == 1:
return value
import torch.distributed as dist
group = self.bootstrap.process_group
values = [value]
dist.broadcast_object_list(values, group=group, group_src=0)
return values[0]
def _tune(self, tensors, capacity: int, native_path: Path):
from flashinfer.moe_ep import MoEEpMegaLayer
backend = replace(
self.backend, megakernel=replace(self.backend.megakernel, knobs="auto")
)
temporary = MoEEpMegaLayer(
bootstrap=self.bootstrap,
fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity),
weights=None,
backend=backend,
)
payload = None
try:
temporary.warmup(tensors)
if self.bootstrap.rank == 0:
try:
payload = json.loads(native_path.read_text())
except (OSError, ValueError):
pass
finally:
temporary.destroy()
payload = self._broadcast(payload)
if not _valid_native_cache(payload):
raise RuntimeError("MegaMoE autotune did not record its selected tactic")
return payload
def _prepare_profile(self, mega, tensors, capacity: int) -> None:
context = _active_context
assert context is not None
profile_key = f"{self._geometry_key}:{capacity}"
profile_inputs = _profile_inputs(
tensors, capacity, self.fleet_params.num_experts
)
native_path = context.staged_profiles.get(profile_key)
needs_staging = native_path is None
if needs_staging:
native_path = (
context.directory / f"profile{len(context.staged_profiles)}.json"
)
with _native_cache(native_path):
if needs_staging:
key = (
self.bootstrap.process_group,
torch.cuda.current_device(),
profile_key,
)
payload = _session_winners.get(key)
if payload is None:
payload = self._broadcast(
context.records.get(profile_key)
if self.bootstrap.rank == 0
else None
)
source = "cache hit"
if not _valid_native_cache(payload):
payload = self._tune(profile_inputs, capacity, native_path)
source = "tuned"
_session_winners[key] = payload
if self.bootstrap.rank == 0:
logger.info(
"MegaMoE %s: capacity=%s, knobs=%s",
source,
capacity,
payload["entries"][0]["knobs"],
)
# The backend owns dtype/layout keys and loads its original payload.
try:
native_path.write_text(json.dumps(payload))
written = True
except OSError:
written = False
self._require_all_ranks(
written,
"Cannot stage MegaMoE's selected tactic for workspace creation",
)
context.staged_profiles[profile_key] = native_path
if context.records.get(profile_key) != payload:
context.records[profile_key] = payload
context.dirty = True
workspace = mega.create_workspace(capacity)
mega.warmup(profile_inputs, workspace=workspace)
self.workspaces[capacity] = workspace
@staticmethod
def _forward(mega, tensors, workspace=None):
if mega.supports_output_view:
return mega.forward(
tensors, workspace=workspace, return_workspace_view=True
)
return mega.forward(tensors, workspace=workspace)
def __call__(self, mega, tensors):
context = _active_context
if context is None and not self.workspaces:
self._fallback_used = True
return self._forward(mega, tensors)
if context is not None and not self.workspaces:
if torch.cuda.is_current_stream_capturing():
raise RuntimeError(
"MegaMoE autotune must complete before CUDA graph capture"
)
if self._fallback_used:
raise RuntimeError(
"MegaMoE autotune must precede its first ordinary forward"
)
import torch.distributed as dist
if (
dist.is_initialized()
and self.bootstrap.world_size != dist.get_world_size()
):
raise RuntimeError(
"FlashInfer MegaMoE startup tuning currently requires the full WORLD process group"
)
# The startup dummy can be much larger than captured decode batches.
# Keep small-N profiles bounded by decode, plus one prefill profile.
capacities = set(_capacities(context.decode_num_tokens))
capacities.add(context.prefill_num_tokens)
for capacity in sorted(capacities):
self._prepare_profile(mega, tensors, capacity)
self._profile_capacities = tuple(sorted(self.workspaces))
count = self._common_tokens(tensors)
index = bisect_left(self._profile_capacities, count)
if index < len(self._profile_capacities):
return self._forward(
mega, tensors, self.workspaces[self._profile_capacities[index]]
)
raise ValueError(
f"MegaMoE received {count} tokens per rank, exceeding its prepared "
f"decode/prefill capacity {max(self.workspaces)}"
)
@@ -19,6 +19,7 @@ import functools
import hashlib
import json
import logging
import uuid
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Optional
@@ -34,8 +35,10 @@ from sglang.srt.runtime_context import (
get_schedule,
get_spec,
max_prefill_buffer_tokens,
max_speculative_num_draft_tokens,
)
from sglang.srt.utils import empty_context, log_info_on_rank0
from sglang.srt.utils.common import get_eager_max_batch_size
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
@@ -94,6 +97,7 @@ def should_run_flashinfer_autotune(
"flashinfer_mxfp4",
"flashinfer_cutedsl",
"flashinfer_cutlass",
"flashinfer_megamoe",
]
from sglang.srt.layers.quantization.fp4_utils import (
@@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path:
/ cache_key
)
cache_dir.mkdir(parents=True, exist_ok=True)
return (
cache_dir
/ f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json"
)
rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}"
if (
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
and not mr.is_draft_worker
):
from sglang.srt.platforms import current_platform
# Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's
# namespaced records currently require one writer per file. Follow the
# P2P cache's device isolation, using physical UUIDs for stable identity
# across CUDA index remapping and restarts. Engines sharing physical GPUs
# must use separate SGLANG_CACHE_DIRs.
rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}"
return cache_dir / f"{rank_key}.json"
def _autotune_tactic_sync_group(
@@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
runs_dir = cache_path.parent / "runs"
runs_dir.mkdir(parents=True, exist_ok=True)
autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}"
autotune_cache = runs_dir / (
f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}"
)
logger.info(
"Running FlashInfer autotune (cache reuse DISABLED via "
"SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s",
@@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
mr.forward_stream.wait_stream(torch.cuda.current_stream())
with torch.get_device_module(mr.device).stream(mr.forward_stream):
from sglang.srt.layers.logits_processor import autotune_dummy_run_mode
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
megamoe_autotune_context,
)
skip_ops = get_flashinfer_autotune_skip_ops(mr)
# autotune(cache=...) clears all file-loaded tactics on entry, which would drop
@@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
tuner = AutoTuner.get()
if reuse_cache and autotune_cache.is_file():
tuner.load_configs(str(autotune_cache))
mega_context = empty_context()
if (
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
and not mr.is_draft_worker
and "flashinfer_megamoe" not in skip_ops
):
# The chunk size is already per DP rank. Prepare one prefill profile
# independently of the optional full-model EXTEND autotune pass.
prefill_num_tokens = (
max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
)
mega_context = megamoe_autotune_context(
decode_num_tokens=max(
mr.max_decode_logits_rows(),
get_eager_max_batch_size(mr.max_running_requests)
* mr.decode_num_tokens_per_req(
num_draft_tokens=max_speculative_num_draft_tokens()
),
),
prefill_num_tokens=prefill_num_tokens,
)
with (
_autotune_process_group(sync_group),
autotune(
@@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
cache=None if reuse_cache else str(autotune_cache),
skip_ops=skip_ops,
),
mega_context,
autotune_dummy_run_mode(run_lm_head=run_lm_head),
):
yield
-1
View File
@@ -644,7 +644,6 @@ class Resources(_FlagGroupBase):
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
tbo_event_pool: dict = msgspec.field(default_factory=dict)
flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
# State capturers (installed by their subsystems when capture is on).
indexer_capturer: Any = None
experts_capturer: Any = None
+5
View File
@@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
elif moe_a2a_backend.is_flashinfer_megamoe():
# MegaMoE's capacity profiles require the same decode graph
# bucket on every EP rank. As for FlashInfer A2A above, this only
# enables DP-count bookkeeping; expert inputs remain local.
return True
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
# Same bookkeeping, for the same reason. Bounding mori's receive
# buffer means baking a fan-in size into a captured graph, and the
@@ -3,6 +3,7 @@ import sys
import types
from pathlib import Path
import pytest
import torch
from sglang.test.ci.ci_register import register_cpu_ci
@@ -69,6 +70,7 @@ def _load_megamoe_module(monkeypatch):
for name, module in fake_modules.items():
monkeypatch.setitem(sys.modules, name, module)
_load_autotune_module(monkeypatch)
module_path = (
Path(__file__).resolve().parents[5]
/ "python/sglang/srt/layers/moe/flashinfer_megamoe.py"
@@ -82,6 +84,269 @@ def _load_megamoe_module(monkeypatch):
return module
def _load_autotune_module(monkeypatch):
name = "sglang.srt.layers.moe.flashinfer_megamoe_autotune"
path = (
Path(__file__).resolve().parents[5]
/ "python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py"
)
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
monkeypatch.setitem(sys.modules, name, module)
spec.loader.exec_module(module)
return module
def _profile_fixture(monkeypatch, world_size=1, num_tokens=3):
from dataclasses import dataclass
module = _load_autotune_module(monkeypatch)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
tuner = types.SimpleNamespace(
get_namespaced_records=lambda _: {},
publish_namespaced_records=lambda *_: None,
)
monkeypatch.setitem(
sys.modules,
"flashinfer.autotuner",
types.SimpleNamespace(AutoTuner=types.SimpleNamespace(get=lambda: tuner)),
)
@dataclass
class Tensors:
hidden_states: torch.Tensor
topk_ids: torch.Tensor
topk_weights: torch.Tensor
fc1_alpha: torch.Tensor
scales: torch.Tensor | None = None
@property
def num_tokens(self):
return self.hidden_states.shape[0]
@dataclass
class Config:
knobs: str | None = None
kernel_name: str = "test_backend"
input_norm_const: float = 1.0
@dataclass
class Fleet:
max_tokens_per_rank: int = 4
num_experts: int = 4
token_hidden_size: int = 4
@dataclass
class Backend:
megakernel: Config
transformed_weights: object
preprocess_weights: bool = False
tensors = Tensors(
torch.ones(num_tokens, 4, dtype=torch.bfloat16),
torch.arange(num_tokens * 2, dtype=torch.int32).reshape(num_tokens, 2) % 4,
torch.full((num_tokens, 2), 0.5, dtype=torch.float32),
torch.ones(4, dtype=torch.float32),
)
forward = module.MegaMoeTunedForward(
types.SimpleNamespace(world_size=world_size, rank=0, process_group=None),
Fleet(),
Backend(Config(), object()),
)
calls = []
class Mega:
supports_output_view = False
def forward(self, inputs, **kwargs):
calls.append((inputs, kwargs.get("workspace")))
return inputs.hidden_states
return module, forward, tensors, Mega(), calls
@pytest.mark.parametrize(
"decode_tokens,expected", [(3, [1, 2, 3, 128]), (8, [1, 2, 4, 8, 128])]
)
def test_megamoe_startup_profiles_are_prepared_once(
monkeypatch, decode_tokens, expected
):
module, forward, tensors, mega, calls = _profile_fixture(monkeypatch, num_tokens=96)
prepared = []
def prepare(layer, inputs, capacity):
prepared.append(capacity)
forward.workspaces[capacity] = object()
monkeypatch.setattr(forward, "_prepare_profile", prepare)
with module.megamoe_autotune_context(decode_tokens, prefill_num_tokens=128):
assert forward(mega, tensors) is tensors.hidden_states
# Bounds may exceed the bootstrap fleet. A large initial dummy must not
# create intermediate prefill profiles.
assert prepared == expected
assert calls[-1][1] is forward.workspaces[128]
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
for count, selected in ((2, 2), (decode_tokens, decode_tokens), (32, 128)):
inputs = module._profile_inputs(tensors, count, num_experts=4)
forward(mega, inputs)
assert prepared == expected
assert calls[-1][1] is forward.workspaces[selected]
with pytest.raises(ValueError, match="exceeding its prepared"):
forward(mega, module._profile_inputs(tensors, 129, num_experts=4))
@pytest.mark.parametrize("local_tokens", [0, 3])
def test_megamoe_profile_selection_uses_all_dp_ranks(monkeypatch, local_tokens):
module, forward, tensors, mega, calls = _profile_fixture(
monkeypatch, world_size=4, num_tokens=local_tokens
)
dp = types.ModuleType("sglang.srt.layers.dp_attention")
dp.get_dp_global_num_tokens = lambda: [0, 1, 3, 7]
monkeypatch.setitem(sys.modules, dp.__name__, dp)
forward.workspaces = {1: object(), 4: object(), 8: object(), 128: object()}
forward._profile_capacities = tuple(forward.workspaces)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
forward(mega, tensors)
assert calls[-1][1] is forward.workspaces[8]
def test_megamoe_without_tuning_preserves_capture_forward(monkeypatch):
module, forward, tensors, mega, calls = _profile_fixture(monkeypatch)
forward(mega, tensors)
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
forward(mega, tensors)
assert [workspace for _, workspace in calls] == [None, None]
@pytest.mark.parametrize("new_process", [False, True])
def test_megamoe_profiles_reuse_cached_winner(monkeypatch, new_process):
import json
import os
from dataclasses import replace
from unittest.mock import Mock
module, first, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=1)
payload = {"version": 1, "entries": [{"knobs": {"native_tactic": 7}}]}
events = []
staged = []
native_paths = []
require_all_ranks = module.MegaMoeTunedForward._require_all_ranks
readiness = Mock()
def require(self, success, message):
readiness(success)
require_all_ranks(self, success, message)
monkeypatch.setattr(module.MegaMoeTunedForward, "_require_all_ranks", require)
class Mega:
supports_output_view = True
def __init__(self, **kwargs):
self.auto = kwargs.get("backend", first.backend).megakernel.knobs == "auto"
def warmup(self, inputs, workspace=None):
if self.auto:
events.append("tune")
Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"]).write_text(
json.dumps(payload)
)
else:
assert workspace is self.workspace
events.append("warmup")
def destroy(self):
events.append("destroy")
def create_workspace(self, capacity):
assert capacity == 1
path = Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"])
native_paths.append(path)
staged.append(json.loads(path.read_text()))
self.workspace = object()
events.append("create")
return self.workspace
def forward(self, inputs, *, workspace, return_workspace_view=False):
assert workspace is self.workspace
assert return_workspace_view
return inputs.hidden_states
fake_moe = types.ModuleType("flashinfer.moe_ep")
fake_moe.MoEEpMegaLayer = Mega
fake_autotuner = types.ModuleType("flashinfer.autotuner")
records = {}
tuner = types.SimpleNamespace(
get_namespaced_records=Mock(
side_effect=lambda namespace: records.get(namespace, {}).copy()
),
publish_namespaced_records=Mock(
side_effect=lambda namespace, values: records.__setitem__(
namespace, values.copy()
)
),
)
fake_autotuner.AutoTuner = types.SimpleNamespace(get=lambda: tuner)
monkeypatch.setitem(sys.modules, "flashinfer", types.ModuleType("flashinfer"))
monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe)
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", fake_autotuner)
monkeypatch.setenv("FLASHINFER_MOE_EP_KNOB_CACHE", "original-cache.json")
owners = []
for phase in range(2):
if owners:
if new_process:
module._session_winners.clear() # Only the loaded cache survives.
else:
records.clear() # A new cache context must preserve pooled choices.
with module.megamoe_autotune_context(
decode_num_tokens=1,
prefill_num_tokens=1,
):
for scale in (1.0, 2.0):
backend = replace(
first.backend,
megakernel=replace(
first.backend.megakernel, input_norm_const=scale
),
)
forward = module.MegaMoeTunedForward(
first.bootstrap, first.fleet_params, backend
)
mega = Mega()
assert forward(mega, tensors) is tensors.hidden_states
owners.append(mega)
assert readiness.call_count == phase + 1
assert native_paths[-2] == native_paths[-1]
assert native_paths[-1].is_file()
assert not native_paths[-1].exists()
assert os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"] == "original-cache.json"
assert list(records["sglang_flashinfer_megamoe"].values()) == [payload]
assert tuner.get_namespaced_records.call_count == phase + 1
assert tuner.publish_namespaced_records.call_count == (1 if new_process else 2)
assert events == ["tune", "destroy"] + ["create", "warmup"] * 4
assert staged == [payload] * 4
assert len({id(owner.workspace) for owner in owners}) == 4
@pytest.mark.parametrize("tokens,capacity", [(0, 8), (3, 8), (3, 1)])
def test_megamoe_profile_inputs_retain_runtime_scales(monkeypatch, tokens, capacity):
module, _, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=tokens)
tensors.scales = torch.ones(tokens, 2)
profile = module._profile_inputs(tensors, capacity, num_experts=4)
assert profile.hidden_states.shape == (capacity, 4)
assert profile.topk_ids.shape == profile.topk_weights.shape == (capacity, 2)
assert profile.hidden_states.dtype == torch.bfloat16
assert profile.topk_ids.dtype == torch.int32
assert profile.topk_weights.dtype == torch.float32
assert profile.fc1_alpha is tensors.fc1_alpha
assert profile.scales.shape == (capacity, 2)
assert torch.all(profile.scales == 1)
assert torch.all((profile.topk_ids >= 0) & (profile.topk_ids < 4))
def test_max_tokens_uses_runtime_context_accessor(monkeypatch):
module = _load_megamoe_module(monkeypatch)
@@ -112,7 +377,7 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
output = torch.randn_like(hidden_states)
class Mega:
_workspace = object()
supports_output_view = False
def forward(self, tensors):
self.tensors = tensors
@@ -140,7 +405,8 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
assert result.hidden_states is output
def test_adapter_requests_workspace_output_view(monkeypatch):
@pytest.mark.parametrize("supports_output_view", [False, True])
def test_adapter_requests_workspace_output_view(monkeypatch, supports_output_view):
module = _load_megamoe_module(monkeypatch)
class FakeMoEEpTensors:
@@ -160,15 +426,13 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
output = torch.randn_like(hidden_states)
class Mega:
supports_output_view = True
_workspace = object()
def forward(self, tensors, *, return_workspace_view=False):
self.tensors = tensors
self.return_workspace_view = return_workspace_view
return output
mega = Mega()
mega.supports_output_view = supports_output_view
dispatch_output = types.SimpleNamespace(
hidden_states=hidden_states,
topk_output=types.SimpleNamespace(
@@ -186,7 +450,7 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
assert result.hidden_states is output
assert mega.tensors.topk_ids.data_ptr() == topk_ids.data_ptr()
assert mega.tensors.topk_ids.dtype == torch.int32
assert mega.return_workspace_view is True
assert mega.return_workspace_view is supports_output_view
def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
@@ -215,6 +479,4 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
if __name__ == "__main__":
import pytest
sys.exit(pytest.main([__file__, "-v"]))
@@ -14,6 +14,7 @@ register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-l
import json
import multiprocessing
import os
import sys
import tempfile
import traceback
import unittest
@@ -65,6 +66,146 @@ class TestAutotuneTacticSyncGroup(CustomTestCase):
self.assertIsNone(_autotune_tactic_sync_group(tp_group))
class TestMegaMoEAutotuneStartup(CustomTestCase):
def test_context_bridges_profile_limits(self):
from sglang.srt.layers.moe import flashinfer_megamoe_autotune as mega_autotune
runner = SimpleNamespace(
device="cuda",
forward_stream=Mock(),
max_decode_logits_rows=Mock(),
decode_num_tokens_per_req=Mock(),
max_running_requests=256,
is_draft_worker=False,
)
general = Mock(side_effect=lambda *_args, **_kwargs: nullcontext())
modules = {
"flashinfer.autotuner": SimpleNamespace(
AutoTuner=SimpleNamespace(
get=lambda: SimpleNamespace(
get_namespaced_records=lambda _: {},
load_configs=lambda _: None,
save_configs=lambda _: None,
publish_namespaced_records=lambda *_: None,
)
),
_collect_metadata=lambda: ENV,
autotune=general,
get_autotune_process_group=lambda: None,
set_autotune_process_group=lambda _: None,
),
"sglang.srt.layers.logits_processor": SimpleNamespace(
autotune_dummy_run_mode=lambda **_: nullcontext()
),
}
# The resolved chunk size is already per DP rank. Disabled chunking
# uses the scheduler's prefill ceiling; skip-op suppresses the bridge.
# (chunk, skip, draft, graph rows, maximum MTP width, decode, prefill)
cases = [
(4096, False, False, 128 * 6, 6, 256 * 6, 4096),
(-1, False, False, 8192, 6, 8192, 16384),
(4096, False, False, 4096, 32, 8192, 4096),
(4096, True, False, 128 * 6, 6, None, None),
(4096, False, True, 128 * 6, 6, None, None),
]
previous = mega_autotune._active_context
with tempfile.TemporaryDirectory() as directory:
cache_path = Path(directory) / "rank.json"
for chunk_size, skip, draft, graph_rows, width, decode, prefill in cases:
runner.is_draft_worker = draft
runner.max_decode_logits_rows.return_value = graph_rows
runner.decode_num_tokens_per_req.return_value = width
skip_ops = {"flashinfer_megamoe"} if skip else set()
with (
self.subTest(chunk_size=chunk_size, skip=skip, draft=draft),
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
patch.dict(sys.modules, modules),
patch.dict(
os.environ,
SGLANG_FLASHINFER_AUTOTUNE_CACHE="1",
),
patch.multiple(
autotune,
flashinfer_autotune_cache_path=lambda _: cache_path,
get_flashinfer_autotune_skip_ops=lambda _: skip_ops,
get_exec=lambda: SimpleNamespace(
moe=SimpleNamespace(moe_runner_backend="flashinfer_megamoe")
),
get_eager_max_batch_size=lambda bs: bs,
max_speculative_num_draft_tokens=lambda: width,
max_prefill_buffer_tokens=lambda: max(chunk_size, 0),
get_schedule=lambda: SimpleNamespace(max_prefill_tokens=16384),
),
patch.object(
autotune.torch.cuda,
"current_stream",
return_value=Mock(),
),
patch.object(
autotune.torch,
"get_device_module",
return_value=SimpleNamespace(stream=lambda _: nullcontext()),
),
):
general.reset_mock()
with autotune.flashinfer_autotune_context(
runner, run_lm_head=False
):
general.assert_called_once_with(
True,
cache=None,
skip_ops=skip_ops,
)
context = mega_autotune._active_context
if prefill is None:
self.assertIs(context, previous)
else:
self.assertIsNot(context, previous)
runner.max_decode_logits_rows.assert_called_with()
runner.decode_num_tokens_per_req.assert_called_with(
num_draft_tokens=width
)
self.assertEqual(context.decode_num_tokens, decode)
self.assertEqual(context.prefill_num_tokens, prefill)
self.assertIs(mega_autotune._active_context, previous)
# Cache-disabled startups in the same second must still write to
# different files, even if they use the same device and rank.
with (
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
patch.dict(sys.modules, modules),
patch.dict(os.environ, SGLANG_FLASHINFER_AUTOTUNE_CACHE="0"),
patch.multiple(
autotune,
flashinfer_autotune_cache_path=lambda _: cache_path,
get_flashinfer_autotune_skip_ops=lambda _: set(),
get_exec=lambda: SimpleNamespace(
moe=SimpleNamespace(moe_runner_backend="triton")
),
),
patch.object(autotune.datetime, "datetime") as clock,
patch.object(
autotune.torch.cuda, "current_stream", return_value=Mock()
),
patch.object(
autotune.torch,
"get_device_module",
return_value=SimpleNamespace(stream=lambda _: nullcontext()),
),
):
clock.now.return_value.strftime.return_value = "20260922_010000"
paths = []
for _ in range(2):
with autotune.flashinfer_autotune_context(
runner, run_lm_head=False
):
paths.append(Path(general.call_args.kwargs["cache"]))
self.assertNotEqual(*paths)
for path in paths:
self.assertEqual(path.parent, cache_path.parent / "runs")
self.assertTrue(path.name.startswith("rank.20260922_010000."))
class TestAutotuneCacheDigest(CustomTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
@@ -249,6 +390,13 @@ class TestAutotuneCachePhases(CustomTestCase):
patch.object(
autotune, "get_flashinfer_autotune_skip_ops", return_value=set()
),
patch.object(
autotune,
"get_exec",
return_value=SimpleNamespace(
moe=SimpleNamespace(moe_runner_backend="flashinfer_cutedsl")
),
),
autotune.envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.override(True),
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
):