diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx
index b7aa1e550b..7bf0292142 100644
--- a/docs/docs/references/environment_variables.mdx
+++ b/docs/docs/references/environment_variables.mdx
@@ -164,7 +164,7 @@ SGLang supports various environment variables that can be used to configure its
| `SGLANG_FLASHINFER_AUTOTUNE_CACHE` |
- Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs. Set to `0` to force re-autotuning on every startup; the fresh result is written to a `runs/..json` sibling file (the canonical cache is left untouched). |
+ Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs, including MegaMoE capacity profiles. Set to `0` to disable disk-cache reuse; the fresh result is written to a `runs/...json` sibling file (the canonical cache is left untouched). Target MegaMoE rank files include the physical GPU UUID. Use separate `SGLANG_CACHE_DIR` roots for engines sharing physical GPUs; a running process retains tactics for already-pooled workspaces. |
`true` |
@@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE |
- Use the CuTe DSL MegaMOE in-kernel FC2 reduction. This can reduce workspace size and improve large-batch performance, but BF16 atomic accumulation is nondeterministic. It is incompatible with quantized combine dtypes. |
+ Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. |
"false" |
diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py
index d4cc184b30..a6a38c32b1 100644
--- a/python/sglang/srt/environ.py
+++ b/python/sglang/srt/environ.py
@@ -1097,9 +1097,10 @@ class Envs:
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
- # cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging
- # region and can win at large batch, but makes the output accumulation order
- # nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility.
+ # cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
+ # FlashInfer's tuning profile may still select external reduction.
+ # BF16 atomic accumulation is nondeterministic -- keep off for
+ # bit-reproducibility.
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False)
# Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE
diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe.py b/python/sglang/srt/layers/moe/flashinfer_megamoe.py
index 1e336f5768..1726338398 100644
--- a/python/sglang/srt/layers/moe/flashinfer_megamoe.py
+++ b/python/sglang/srt/layers/moe/flashinfer_megamoe.py
@@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor:
def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]:
- import inspect
-
- if "return_workspace_view" in inspect.signature(mega.forward).parameters:
+ if mega.supports_output_view:
return _forward_megamoe_with_workspace_view
return _forward_megamoe_legacy
@@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer(
layer.hidden_size,
)
+ process_group = None
+ if world_size > 1:
+ from sglang.srt.runtime_context import get_parallel
+
+ process_group = get_parallel().moe_ep_group.device_group
+ bootstrap = BootstrapConfig(
+ world_size=world_size,
+ rank=rank,
+ device=torch.cuda.current_device(),
+ process_group=process_group,
+ )
+ fleet_params = FleetParams(
+ num_experts=layer.num_experts,
+ max_tokens_per_rank=max_tokens_per_rank,
+ token_hidden_size=layer.hidden_size,
+ )
+ backend = MegaConfig(
+ megakernel=megakernel_config,
+ preprocess_weights=False,
+ transformed_weights=transformed_weights,
+ )
mega = MoEEpMegaLayer(
- bootstrap=BootstrapConfig(
- world_size=world_size, rank=rank, device=torch.cuda.current_device()
- ),
- fleet_params=FleetParams(
- num_experts=layer.num_experts,
- max_tokens_per_rank=max_tokens_per_rank,
- token_hidden_size=layer.hidden_size,
- ),
+ bootstrap=bootstrap,
+ fleet_params=fleet_params,
# weights already preprocessed in prepare_*; with transformed_weights set
# the kernel never reads `weights` (see MoEEpMegaLayer), so pass None.
weights=None,
- backend=MegaConfig(
- megakernel=megakernel_config,
- preprocess_weights=False,
- transformed_weights=transformed_weights,
- ),
+ backend=backend,
)
layer._flashinfer_megamoe_layer = mega
- layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
+ if hasattr(megakernel_config, "knobs"):
+ from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
+ MegaMoeTunedForward,
+ )
+
+ layer._flashinfer_megamoe_forward = MegaMoeTunedForward(
+ bootstrap, fleet_params, backend
+ )
+ else:
+ layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
return mega
@@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
apply_topk_in_fc1=True,
- in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
+ enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
input_norm_const=input_norm_const,
- fc1_alpha=layer.g1_alphas,
- fc2_alpha=layer.g2_alphas,
- fc1_norm_const=layer.w2_input_scale_quant,
),
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
@@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
top_k=layer.top_k,
kind="mxfp8_e4m3",
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
- in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
+ enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
),
w13_scale=layer.w13_weight_scale_inv,
w2_scale=layer.w2_weight_scale_inv,
@@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
)
-def _ensure_shared_workspace(mega: Any) -> None:
- """Share this layer's workspace across MegaMOE layers with identical
- fleet/kernel geometry.
- FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const
- tensor identity, but sglang binds distinct tensor objects per layer, so
- its pool never hits across layers; key on geometry instead, since those
- values are re-staged into the workspace per forward rather than baked
- into the compiled kernel. Creation is collective, so it must only happen
- on a layer's first forward, under warmup's cross-rank lockstep.
- """
- if mega._workspace is not None:
- return
- fp = mega._fleet_params
- kc = mega._megakernel_config
- mc = mega._mega_config
- from sglang.srt.runtime_context import get_resources
-
- key = (
- getattr(kc, "kernel_name", kc.__class__.__name__),
- mega._bootstrap.world_size,
- fp.num_experts,
- fp.max_tokens_per_rank,
- fp.token_hidden_size,
- kc.top_k,
- kc.intermediate_size,
- getattr(kc, "gate_up_clamp", None),
- getattr(kc, "activation_clamp", None),
- getattr(kc, "apply_topk_in_fc1", None),
- getattr(kc, "kind", None),
- getattr(kc, "in_kernel_fc2_reduce", None),
- getattr(kc, "combine_dtype", None),
- getattr(kc, "token_back_by_dispatch", None),
- getattr(kc, "fast_math", None),
- mc.quantize_input,
- )
- workspaces = get_resources().flashinfer_megamoe_workspaces
- shared = workspaces.get(key)
- if shared is None:
- workspaces[key] = mega._ensure_workspace()
- else:
- mega._workspace = shared
-
-
@register_fused_func("flashinfer_megamoe", "flashinfer_megamoe")
def run_flashinfer_megamoe(
dispatch_output: DispatchOutput,
@@ -599,7 +571,6 @@ def run_flashinfer_megamoe(
topk_weights = topk_output.topk_weights
topk_ids = topk_output.topk_ids
mega = quant_info.mega
- _ensure_shared_workspace(mega)
t = MoEEpTensors(
hidden_states=x.to(torch.bfloat16),
@@ -607,6 +578,7 @@ def run_flashinfer_megamoe(
# directly into its final int64 workspace buffer. Keep this path copy-free.
topk_ids=topk_ids,
topk_weights=topk_weights.to(torch.float32),
+ # Stage per-layer scales on every call so FlashInfer can pool workspaces.
fc1_alpha=quant_info.fc1_alpha,
fc2_alpha=quant_info.fc2_alpha,
fc1_norm_const=quant_info.fc1_norm_const,
diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py b/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py
new file mode 100644
index 0000000000..19fb1eebf2
--- /dev/null
+++ b/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py
@@ -0,0 +1,366 @@
+# Copyright 2023-2026 SGLang Team
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from bisect import bisect_left
+from contextlib import contextmanager
+from dataclasses import dataclass, field, fields, replace
+from functools import cached_property
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class _AutotuneContext:
+ decode_num_tokens: int
+ prefill_num_tokens: int
+ directory: Path
+ records: dict
+ staged_profiles: dict[str, Path] = field(default_factory=dict)
+ dirty: bool = False
+
+
+_active_context: _AutotuneContext | None = None
+# Native workspace pooling spans startup contexts. Keep their
+# selected tactics consistent too; a cache-path change must not retune storage
+# already captured by another layer. These payloads own no weights or tensors.
+_session_winners: dict[tuple[Any, int, str], dict] = {}
+_CACHE_NAMESPACE = "sglang_flashinfer_megamoe"
+
+
+@contextmanager
+def megamoe_autotune_context(
+ decode_num_tokens: int,
+ prefill_num_tokens: int,
+):
+ """Prepare profiles inside the engine's pre-capture FlashInfer autotune pass."""
+ from flashinfer.autotuner import AutoTuner
+
+ global _active_context
+ tuner = AutoTuner.get()
+ previous = _active_context
+ with TemporaryDirectory(prefix="sglang-megamoe-") as directory:
+ context = _AutotuneContext(
+ decode_num_tokens,
+ prefill_num_tokens,
+ Path(directory),
+ tuner.get_namespaced_records(_CACHE_NAMESPACE),
+ )
+ _active_context = context
+ try:
+ yield
+ finally:
+ _active_context = previous
+ if context.dirty:
+ tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records)
+
+
+@contextmanager
+def _native_cache(path: Path):
+ # Native AUTO exposes its winner through this file; generic autotune owns
+ # persistence. Each geometry is staged once per startup context.
+ name = "FLASHINFER_MOE_EP_KNOB_CACHE"
+ previous = os.environ.get(name)
+ os.environ[name] = str(path)
+ try:
+ yield
+ finally:
+ if previous is None:
+ os.environ.pop(name, None)
+ else:
+ os.environ[name] = previous
+
+
+def _capacities(limit: int) -> list[int]:
+ limit = max(1, limit)
+ return sorted({1 << i for i in range(limit.bit_length())} | {limit})
+
+
+def _profile_inputs(tensors, capacity: int, num_experts: int):
+ count = tensors.num_tokens
+ if count:
+
+ def resize(tensor):
+ if capacity <= count:
+ return tensor[:capacity]
+ return tensor.repeat(
+ ((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1)
+ )[:capacity]
+
+ hidden = resize(tensors.hidden_states)
+ ids = resize(tensors.topk_ids)
+ scores = resize(tensors.topk_weights)
+ scales = resize(tensors.scales) if tensors.scales is not None else None
+ else:
+ # Idle ranks still enter every collective tuning iteration.
+ hidden = tensors.hidden_states.new_zeros(
+ (capacity, tensors.hidden_states.shape[1])
+ )
+ top_k = tensors.topk_ids.shape[1]
+ ids = (
+ torch.arange(
+ capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device
+ ).reshape(capacity, top_k)
+ % num_experts
+ )
+ scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k)
+ scales = (
+ tensors.scales.new_ones((capacity, *tensors.scales.shape[1:]))
+ if tensors.scales is not None
+ else None
+ )
+ return replace(
+ tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales
+ )
+
+
+def _valid_native_cache(payload) -> bool:
+ if not isinstance(payload, dict) or payload.get("version") != 1:
+ return False
+ entries = payload.get("entries")
+ return (
+ isinstance(entries, list)
+ and len(entries) == 1
+ and isinstance(entries[0], dict)
+ and isinstance(entries[0].get("knobs"), dict)
+ and bool(entries[0]["knobs"])
+ )
+
+
+class MegaMoeTunedForward:
+ """Own public workspace handles; FlashInfer pools their storage across layers.
+
+ The caller installs this for backends with prepared weights and default
+ ``knobs=None``. Preparation must precede ordinary forwarding and capture.
+ """
+
+ def __init__(self, bootstrap, fleet_params, backend):
+ if backend.megakernel.knobs is not None:
+ raise ValueError("MegaMoE startup tuning requires knobs=None")
+ if backend.preprocess_weights or backend.transformed_weights is None:
+ raise ValueError("MegaMoE startup tuning requires prepared weights")
+ self.bootstrap = bootstrap
+ self.fleet_params = fleet_params
+ self.backend = backend
+ self.workspaces: dict[int, Any] = {}
+ self._fallback_used = False
+ self._profile_capacities: tuple[int, ...] = ()
+
+ def _common_tokens(self, tensors) -> int:
+ if self.bootstrap.world_size == 1:
+ return tensors.num_tokens
+ from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
+
+ counts = get_dp_global_num_tokens()
+ if counts is None:
+ raise RuntimeError(
+ "MegaMoE workspace selection requires synchronized DP token counts"
+ )
+ return max(counts, default=0)
+
+ @cached_property
+ def _geometry_key(self) -> str:
+ config = {}
+ for item in fields(self.backend.megakernel):
+ # Per-layer scale values do not change the pooled kernel geometry.
+ if item.name in (
+ "knobs",
+ "input_norm_const",
+ "fc1_norm_const",
+ "fc1_alpha",
+ "fc2_alpha",
+ ):
+ continue
+ value = getattr(self.backend.megakernel, item.name)
+ if not isinstance(value, torch.Tensor):
+ config[item.name] = value
+ # The enclosing flashinfer.autotune context validates runtime/compiler
+ # metadata. This key only adds the MegaMoE configuration and geometry.
+ return json.dumps(
+ {
+ "torch": torch.__version__,
+ "mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"),
+ "config": config,
+ "world_size": self.bootstrap.world_size,
+ "hidden": self.fleet_params.token_hidden_size,
+ "num_experts": self.fleet_params.num_experts,
+ },
+ sort_keys=True,
+ )
+
+ def _require_all_ranks(self, success: bool, message: str) -> None:
+ if self.bootstrap.world_size > 1:
+ import torch.distributed as dist
+
+ ready = torch.tensor(int(success), dtype=torch.int32, device="cuda")
+ dist.all_reduce(
+ ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group
+ )
+ success = bool(ready.item())
+ if not success:
+ raise RuntimeError(message)
+
+ def _broadcast(self, value):
+ if self.bootstrap.world_size == 1:
+ return value
+ import torch.distributed as dist
+
+ group = self.bootstrap.process_group
+ values = [value]
+ dist.broadcast_object_list(values, group=group, group_src=0)
+ return values[0]
+
+ def _tune(self, tensors, capacity: int, native_path: Path):
+ from flashinfer.moe_ep import MoEEpMegaLayer
+
+ backend = replace(
+ self.backend, megakernel=replace(self.backend.megakernel, knobs="auto")
+ )
+ temporary = MoEEpMegaLayer(
+ bootstrap=self.bootstrap,
+ fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity),
+ weights=None,
+ backend=backend,
+ )
+ payload = None
+ try:
+ temporary.warmup(tensors)
+ if self.bootstrap.rank == 0:
+ try:
+ payload = json.loads(native_path.read_text())
+ except (OSError, ValueError):
+ pass
+ finally:
+ temporary.destroy()
+ payload = self._broadcast(payload)
+ if not _valid_native_cache(payload):
+ raise RuntimeError("MegaMoE autotune did not record its selected tactic")
+ return payload
+
+ def _prepare_profile(self, mega, tensors, capacity: int) -> None:
+ context = _active_context
+ assert context is not None
+ profile_key = f"{self._geometry_key}:{capacity}"
+ profile_inputs = _profile_inputs(
+ tensors, capacity, self.fleet_params.num_experts
+ )
+ native_path = context.staged_profiles.get(profile_key)
+ needs_staging = native_path is None
+ if needs_staging:
+ native_path = (
+ context.directory / f"profile{len(context.staged_profiles)}.json"
+ )
+ with _native_cache(native_path):
+ if needs_staging:
+ key = (
+ self.bootstrap.process_group,
+ torch.cuda.current_device(),
+ profile_key,
+ )
+ payload = _session_winners.get(key)
+ if payload is None:
+ payload = self._broadcast(
+ context.records.get(profile_key)
+ if self.bootstrap.rank == 0
+ else None
+ )
+ source = "cache hit"
+ if not _valid_native_cache(payload):
+ payload = self._tune(profile_inputs, capacity, native_path)
+ source = "tuned"
+ _session_winners[key] = payload
+ if self.bootstrap.rank == 0:
+ logger.info(
+ "MegaMoE %s: capacity=%s, knobs=%s",
+ source,
+ capacity,
+ payload["entries"][0]["knobs"],
+ )
+ # The backend owns dtype/layout keys and loads its original payload.
+ try:
+ native_path.write_text(json.dumps(payload))
+ written = True
+ except OSError:
+ written = False
+ self._require_all_ranks(
+ written,
+ "Cannot stage MegaMoE's selected tactic for workspace creation",
+ )
+ context.staged_profiles[profile_key] = native_path
+ if context.records.get(profile_key) != payload:
+ context.records[profile_key] = payload
+ context.dirty = True
+ workspace = mega.create_workspace(capacity)
+ mega.warmup(profile_inputs, workspace=workspace)
+ self.workspaces[capacity] = workspace
+
+ @staticmethod
+ def _forward(mega, tensors, workspace=None):
+ if mega.supports_output_view:
+ return mega.forward(
+ tensors, workspace=workspace, return_workspace_view=True
+ )
+ return mega.forward(tensors, workspace=workspace)
+
+ def __call__(self, mega, tensors):
+ context = _active_context
+ if context is None and not self.workspaces:
+ self._fallback_used = True
+ return self._forward(mega, tensors)
+ if context is not None and not self.workspaces:
+ if torch.cuda.is_current_stream_capturing():
+ raise RuntimeError(
+ "MegaMoE autotune must complete before CUDA graph capture"
+ )
+ if self._fallback_used:
+ raise RuntimeError(
+ "MegaMoE autotune must precede its first ordinary forward"
+ )
+ import torch.distributed as dist
+
+ if (
+ dist.is_initialized()
+ and self.bootstrap.world_size != dist.get_world_size()
+ ):
+ raise RuntimeError(
+ "FlashInfer MegaMoE startup tuning currently requires the full WORLD process group"
+ )
+ # The startup dummy can be much larger than captured decode batches.
+ # Keep small-N profiles bounded by decode, plus one prefill profile.
+ capacities = set(_capacities(context.decode_num_tokens))
+ capacities.add(context.prefill_num_tokens)
+ for capacity in sorted(capacities):
+ self._prepare_profile(mega, tensors, capacity)
+ self._profile_capacities = tuple(sorted(self.workspaces))
+
+ count = self._common_tokens(tensors)
+ index = bisect_left(self._profile_capacities, count)
+ if index < len(self._profile_capacities):
+ return self._forward(
+ mega, tensors, self.workspaces[self._profile_capacities[index]]
+ )
+ raise ValueError(
+ f"MegaMoE received {count} tokens per rank, exceeding its prepared "
+ f"decode/prefill capacity {max(self.workspaces)}"
+ )
diff --git a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py
index 794fdaffd1..c8d2f35519 100644
--- a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py
+++ b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py
@@ -19,6 +19,7 @@ import functools
import hashlib
import json
import logging
+import uuid
from pathlib import Path
from typing import TYPE_CHECKING, Callable, Optional
@@ -34,8 +35,10 @@ from sglang.srt.runtime_context import (
get_schedule,
get_spec,
max_prefill_buffer_tokens,
+ max_speculative_num_draft_tokens,
)
from sglang.srt.utils import empty_context, log_info_on_rank0
+from sglang.srt.utils.common import get_eager_max_batch_size
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
@@ -94,6 +97,7 @@ def should_run_flashinfer_autotune(
"flashinfer_mxfp4",
"flashinfer_cutedsl",
"flashinfer_cutlass",
+ "flashinfer_megamoe",
]
from sglang.srt.layers.quantization.fp4_utils import (
@@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path:
/ cache_key
)
cache_dir.mkdir(parents=True, exist_ok=True)
- return (
- cache_dir
- / f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json"
- )
+ rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}"
+ if (
+ get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
+ and not mr.is_draft_worker
+ ):
+ from sglang.srt.platforms import current_platform
+
+ # Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's
+ # namespaced records currently require one writer per file. Follow the
+ # P2P cache's device isolation, using physical UUIDs for stable identity
+ # across CUDA index remapping and restarts. Engines sharing physical GPUs
+ # must use separate SGLANG_CACHE_DIRs.
+ rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}"
+ return cache_dir / f"{rank_key}.json"
def _autotune_tactic_sync_group(
@@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
runs_dir = cache_path.parent / "runs"
runs_dir.mkdir(parents=True, exist_ok=True)
- autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}"
+ autotune_cache = runs_dir / (
+ f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}"
+ )
logger.info(
"Running FlashInfer autotune (cache reuse DISABLED via "
"SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s",
@@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
mr.forward_stream.wait_stream(torch.cuda.current_stream())
with torch.get_device_module(mr.device).stream(mr.forward_stream):
from sglang.srt.layers.logits_processor import autotune_dummy_run_mode
+ from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
+ megamoe_autotune_context,
+ )
skip_ops = get_flashinfer_autotune_skip_ops(mr)
# autotune(cache=...) clears all file-loaded tactics on entry, which would drop
@@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
tuner = AutoTuner.get()
if reuse_cache and autotune_cache.is_file():
tuner.load_configs(str(autotune_cache))
+ mega_context = empty_context()
+ if (
+ get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
+ and not mr.is_draft_worker
+ and "flashinfer_megamoe" not in skip_ops
+ ):
+ # The chunk size is already per DP rank. Prepare one prefill profile
+ # independently of the optional full-model EXTEND autotune pass.
+ prefill_num_tokens = (
+ max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
+ )
+ mega_context = megamoe_autotune_context(
+ decode_num_tokens=max(
+ mr.max_decode_logits_rows(),
+ get_eager_max_batch_size(mr.max_running_requests)
+ * mr.decode_num_tokens_per_req(
+ num_draft_tokens=max_speculative_num_draft_tokens()
+ ),
+ ),
+ prefill_num_tokens=prefill_num_tokens,
+ )
with (
_autotune_process_group(sync_group),
autotune(
@@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
cache=None if reuse_cache else str(autotune_cache),
skip_ops=skip_ops,
),
+ mega_context,
autotune_dummy_run_mode(run_lm_head=run_lm_head),
):
yield
diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py
index bcea02a32d..0e64489406 100644
--- a/python/sglang/srt/runtime_context.py
+++ b/python/sglang/srt/runtime_context.py
@@ -644,7 +644,6 @@ class Resources(_FlagGroupBase):
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
tbo_event_pool: dict = msgspec.field(default_factory=dict)
- flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
# State capturers (installed by their subsystems when capture is on).
indexer_capturer: Any = None
experts_capturer: Any = None
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index fa07444af5..7ac94f1ed7 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
+ elif moe_a2a_backend.is_flashinfer_megamoe():
+ # MegaMoE's capacity profiles require the same decode graph
+ # bucket on every EP rank. As for FlashInfer A2A above, this only
+ # enables DP-count bookkeeping; expert inputs remain local.
+ return True
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
# Same bookkeeping, for the same reason. Bounding mori's receive
# buffer means baking a fan-in size into a captured graph, and the
diff --git a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
index cd73104ed9..640125fc18 100644
--- a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
+++ b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py
@@ -3,6 +3,7 @@ import sys
import types
from pathlib import Path
+import pytest
import torch
from sglang.test.ci.ci_register import register_cpu_ci
@@ -69,6 +70,7 @@ def _load_megamoe_module(monkeypatch):
for name, module in fake_modules.items():
monkeypatch.setitem(sys.modules, name, module)
+ _load_autotune_module(monkeypatch)
module_path = (
Path(__file__).resolve().parents[5]
/ "python/sglang/srt/layers/moe/flashinfer_megamoe.py"
@@ -82,6 +84,269 @@ def _load_megamoe_module(monkeypatch):
return module
+def _load_autotune_module(monkeypatch):
+ name = "sglang.srt.layers.moe.flashinfer_megamoe_autotune"
+ path = (
+ Path(__file__).resolve().parents[5]
+ / "python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py"
+ )
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ monkeypatch.setitem(sys.modules, name, module)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _profile_fixture(monkeypatch, world_size=1, num_tokens=3):
+ from dataclasses import dataclass
+
+ module = _load_autotune_module(monkeypatch)
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ tuner = types.SimpleNamespace(
+ get_namespaced_records=lambda _: {},
+ publish_namespaced_records=lambda *_: None,
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "flashinfer.autotuner",
+ types.SimpleNamespace(AutoTuner=types.SimpleNamespace(get=lambda: tuner)),
+ )
+
+ @dataclass
+ class Tensors:
+ hidden_states: torch.Tensor
+ topk_ids: torch.Tensor
+ topk_weights: torch.Tensor
+ fc1_alpha: torch.Tensor
+ scales: torch.Tensor | None = None
+
+ @property
+ def num_tokens(self):
+ return self.hidden_states.shape[0]
+
+ @dataclass
+ class Config:
+ knobs: str | None = None
+ kernel_name: str = "test_backend"
+ input_norm_const: float = 1.0
+
+ @dataclass
+ class Fleet:
+ max_tokens_per_rank: int = 4
+ num_experts: int = 4
+ token_hidden_size: int = 4
+
+ @dataclass
+ class Backend:
+ megakernel: Config
+ transformed_weights: object
+ preprocess_weights: bool = False
+
+ tensors = Tensors(
+ torch.ones(num_tokens, 4, dtype=torch.bfloat16),
+ torch.arange(num_tokens * 2, dtype=torch.int32).reshape(num_tokens, 2) % 4,
+ torch.full((num_tokens, 2), 0.5, dtype=torch.float32),
+ torch.ones(4, dtype=torch.float32),
+ )
+ forward = module.MegaMoeTunedForward(
+ types.SimpleNamespace(world_size=world_size, rank=0, process_group=None),
+ Fleet(),
+ Backend(Config(), object()),
+ )
+ calls = []
+
+ class Mega:
+ supports_output_view = False
+
+ def forward(self, inputs, **kwargs):
+ calls.append((inputs, kwargs.get("workspace")))
+ return inputs.hidden_states
+
+ return module, forward, tensors, Mega(), calls
+
+
+@pytest.mark.parametrize(
+ "decode_tokens,expected", [(3, [1, 2, 3, 128]), (8, [1, 2, 4, 8, 128])]
+)
+def test_megamoe_startup_profiles_are_prepared_once(
+ monkeypatch, decode_tokens, expected
+):
+ module, forward, tensors, mega, calls = _profile_fixture(monkeypatch, num_tokens=96)
+ prepared = []
+
+ def prepare(layer, inputs, capacity):
+ prepared.append(capacity)
+ forward.workspaces[capacity] = object()
+
+ monkeypatch.setattr(forward, "_prepare_profile", prepare)
+ with module.megamoe_autotune_context(decode_tokens, prefill_num_tokens=128):
+ assert forward(mega, tensors) is tensors.hidden_states
+ # Bounds may exceed the bootstrap fleet. A large initial dummy must not
+ # create intermediate prefill profiles.
+ assert prepared == expected
+ assert calls[-1][1] is forward.workspaces[128]
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
+ for count, selected in ((2, 2), (decode_tokens, decode_tokens), (32, 128)):
+ inputs = module._profile_inputs(tensors, count, num_experts=4)
+ forward(mega, inputs)
+ assert prepared == expected
+ assert calls[-1][1] is forward.workspaces[selected]
+ with pytest.raises(ValueError, match="exceeding its prepared"):
+ forward(mega, module._profile_inputs(tensors, 129, num_experts=4))
+
+
+@pytest.mark.parametrize("local_tokens", [0, 3])
+def test_megamoe_profile_selection_uses_all_dp_ranks(monkeypatch, local_tokens):
+ module, forward, tensors, mega, calls = _profile_fixture(
+ monkeypatch, world_size=4, num_tokens=local_tokens
+ )
+ dp = types.ModuleType("sglang.srt.layers.dp_attention")
+ dp.get_dp_global_num_tokens = lambda: [0, 1, 3, 7]
+ monkeypatch.setitem(sys.modules, dp.__name__, dp)
+ forward.workspaces = {1: object(), 4: object(), 8: object(), 128: object()}
+ forward._profile_capacities = tuple(forward.workspaces)
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
+ forward(mega, tensors)
+ assert calls[-1][1] is forward.workspaces[8]
+
+
+def test_megamoe_without_tuning_preserves_capture_forward(monkeypatch):
+ module, forward, tensors, mega, calls = _profile_fixture(monkeypatch)
+ forward(mega, tensors)
+ monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
+ forward(mega, tensors)
+ assert [workspace for _, workspace in calls] == [None, None]
+
+
+@pytest.mark.parametrize("new_process", [False, True])
+def test_megamoe_profiles_reuse_cached_winner(monkeypatch, new_process):
+ import json
+ import os
+ from dataclasses import replace
+ from unittest.mock import Mock
+
+ module, first, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=1)
+ payload = {"version": 1, "entries": [{"knobs": {"native_tactic": 7}}]}
+ events = []
+ staged = []
+ native_paths = []
+ require_all_ranks = module.MegaMoeTunedForward._require_all_ranks
+ readiness = Mock()
+
+ def require(self, success, message):
+ readiness(success)
+ require_all_ranks(self, success, message)
+
+ monkeypatch.setattr(module.MegaMoeTunedForward, "_require_all_ranks", require)
+
+ class Mega:
+ supports_output_view = True
+
+ def __init__(self, **kwargs):
+ self.auto = kwargs.get("backend", first.backend).megakernel.knobs == "auto"
+
+ def warmup(self, inputs, workspace=None):
+ if self.auto:
+ events.append("tune")
+ Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"]).write_text(
+ json.dumps(payload)
+ )
+ else:
+ assert workspace is self.workspace
+ events.append("warmup")
+
+ def destroy(self):
+ events.append("destroy")
+
+ def create_workspace(self, capacity):
+ assert capacity == 1
+ path = Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"])
+ native_paths.append(path)
+ staged.append(json.loads(path.read_text()))
+ self.workspace = object()
+ events.append("create")
+ return self.workspace
+
+ def forward(self, inputs, *, workspace, return_workspace_view=False):
+ assert workspace is self.workspace
+ assert return_workspace_view
+ return inputs.hidden_states
+
+ fake_moe = types.ModuleType("flashinfer.moe_ep")
+ fake_moe.MoEEpMegaLayer = Mega
+ fake_autotuner = types.ModuleType("flashinfer.autotuner")
+ records = {}
+ tuner = types.SimpleNamespace(
+ get_namespaced_records=Mock(
+ side_effect=lambda namespace: records.get(namespace, {}).copy()
+ ),
+ publish_namespaced_records=Mock(
+ side_effect=lambda namespace, values: records.__setitem__(
+ namespace, values.copy()
+ )
+ ),
+ )
+ fake_autotuner.AutoTuner = types.SimpleNamespace(get=lambda: tuner)
+ monkeypatch.setitem(sys.modules, "flashinfer", types.ModuleType("flashinfer"))
+ monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe)
+ monkeypatch.setitem(sys.modules, "flashinfer.autotuner", fake_autotuner)
+ monkeypatch.setenv("FLASHINFER_MOE_EP_KNOB_CACHE", "original-cache.json")
+
+ owners = []
+ for phase in range(2):
+ if owners:
+ if new_process:
+ module._session_winners.clear() # Only the loaded cache survives.
+ else:
+ records.clear() # A new cache context must preserve pooled choices.
+ with module.megamoe_autotune_context(
+ decode_num_tokens=1,
+ prefill_num_tokens=1,
+ ):
+ for scale in (1.0, 2.0):
+ backend = replace(
+ first.backend,
+ megakernel=replace(
+ first.backend.megakernel, input_norm_const=scale
+ ),
+ )
+ forward = module.MegaMoeTunedForward(
+ first.bootstrap, first.fleet_params, backend
+ )
+ mega = Mega()
+ assert forward(mega, tensors) is tensors.hidden_states
+ owners.append(mega)
+ assert readiness.call_count == phase + 1
+ assert native_paths[-2] == native_paths[-1]
+ assert native_paths[-1].is_file()
+ assert not native_paths[-1].exists()
+ assert os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"] == "original-cache.json"
+ assert list(records["sglang_flashinfer_megamoe"].values()) == [payload]
+ assert tuner.get_namespaced_records.call_count == phase + 1
+
+ assert tuner.publish_namespaced_records.call_count == (1 if new_process else 2)
+ assert events == ["tune", "destroy"] + ["create", "warmup"] * 4
+ assert staged == [payload] * 4
+ assert len({id(owner.workspace) for owner in owners}) == 4
+
+
+@pytest.mark.parametrize("tokens,capacity", [(0, 8), (3, 8), (3, 1)])
+def test_megamoe_profile_inputs_retain_runtime_scales(monkeypatch, tokens, capacity):
+ module, _, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=tokens)
+ tensors.scales = torch.ones(tokens, 2)
+ profile = module._profile_inputs(tensors, capacity, num_experts=4)
+ assert profile.hidden_states.shape == (capacity, 4)
+ assert profile.topk_ids.shape == profile.topk_weights.shape == (capacity, 2)
+ assert profile.hidden_states.dtype == torch.bfloat16
+ assert profile.topk_ids.dtype == torch.int32
+ assert profile.topk_weights.dtype == torch.float32
+ assert profile.fc1_alpha is tensors.fc1_alpha
+ assert profile.scales.shape == (capacity, 2)
+ assert torch.all(profile.scales == 1)
+ assert torch.all((profile.topk_ids >= 0) & (profile.topk_ids < 4))
+
+
def test_max_tokens_uses_runtime_context_accessor(monkeypatch):
module = _load_megamoe_module(monkeypatch)
@@ -112,7 +377,7 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
output = torch.randn_like(hidden_states)
class Mega:
- _workspace = object()
+ supports_output_view = False
def forward(self, tensors):
self.tensors = tensors
@@ -140,7 +405,8 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
assert result.hidden_states is output
-def test_adapter_requests_workspace_output_view(monkeypatch):
+@pytest.mark.parametrize("supports_output_view", [False, True])
+def test_adapter_requests_workspace_output_view(monkeypatch, supports_output_view):
module = _load_megamoe_module(monkeypatch)
class FakeMoEEpTensors:
@@ -160,15 +426,13 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
output = torch.randn_like(hidden_states)
class Mega:
- supports_output_view = True
- _workspace = object()
-
def forward(self, tensors, *, return_workspace_view=False):
self.tensors = tensors
self.return_workspace_view = return_workspace_view
return output
mega = Mega()
+ mega.supports_output_view = supports_output_view
dispatch_output = types.SimpleNamespace(
hidden_states=hidden_states,
topk_output=types.SimpleNamespace(
@@ -186,7 +450,7 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
assert result.hidden_states is output
assert mega.tensors.topk_ids.data_ptr() == topk_ids.data_ptr()
assert mega.tensors.topk_ids.dtype == torch.int32
- assert mega.return_workspace_view is True
+ assert mega.return_workspace_view is supports_output_view
def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
@@ -215,6 +479,4 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
if __name__ == "__main__":
- import pytest
-
sys.exit(pytest.main([__file__, "-v"]))
diff --git a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py
index c8151e4455..14ed53a32a 100644
--- a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py
+++ b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py
@@ -14,6 +14,7 @@ register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-l
import json
import multiprocessing
import os
+import sys
import tempfile
import traceback
import unittest
@@ -65,6 +66,146 @@ class TestAutotuneTacticSyncGroup(CustomTestCase):
self.assertIsNone(_autotune_tactic_sync_group(tp_group))
+class TestMegaMoEAutotuneStartup(CustomTestCase):
+ def test_context_bridges_profile_limits(self):
+ from sglang.srt.layers.moe import flashinfer_megamoe_autotune as mega_autotune
+
+ runner = SimpleNamespace(
+ device="cuda",
+ forward_stream=Mock(),
+ max_decode_logits_rows=Mock(),
+ decode_num_tokens_per_req=Mock(),
+ max_running_requests=256,
+ is_draft_worker=False,
+ )
+ general = Mock(side_effect=lambda *_args, **_kwargs: nullcontext())
+ modules = {
+ "flashinfer.autotuner": SimpleNamespace(
+ AutoTuner=SimpleNamespace(
+ get=lambda: SimpleNamespace(
+ get_namespaced_records=lambda _: {},
+ load_configs=lambda _: None,
+ save_configs=lambda _: None,
+ publish_namespaced_records=lambda *_: None,
+ )
+ ),
+ _collect_metadata=lambda: ENV,
+ autotune=general,
+ get_autotune_process_group=lambda: None,
+ set_autotune_process_group=lambda _: None,
+ ),
+ "sglang.srt.layers.logits_processor": SimpleNamespace(
+ autotune_dummy_run_mode=lambda **_: nullcontext()
+ ),
+ }
+ # The resolved chunk size is already per DP rank. Disabled chunking
+ # uses the scheduler's prefill ceiling; skip-op suppresses the bridge.
+ # (chunk, skip, draft, graph rows, maximum MTP width, decode, prefill)
+ cases = [
+ (4096, False, False, 128 * 6, 6, 256 * 6, 4096),
+ (-1, False, False, 8192, 6, 8192, 16384),
+ (4096, False, False, 4096, 32, 8192, 4096),
+ (4096, True, False, 128 * 6, 6, None, None),
+ (4096, False, True, 128 * 6, 6, None, None),
+ ]
+ previous = mega_autotune._active_context
+ with tempfile.TemporaryDirectory() as directory:
+ cache_path = Path(directory) / "rank.json"
+ for chunk_size, skip, draft, graph_rows, width, decode, prefill in cases:
+ runner.is_draft_worker = draft
+ runner.max_decode_logits_rows.return_value = graph_rows
+ runner.decode_num_tokens_per_req.return_value = width
+ skip_ops = {"flashinfer_megamoe"} if skip else set()
+ with (
+ self.subTest(chunk_size=chunk_size, skip=skip, draft=draft),
+ get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
+ patch.dict(sys.modules, modules),
+ patch.dict(
+ os.environ,
+ SGLANG_FLASHINFER_AUTOTUNE_CACHE="1",
+ ),
+ patch.multiple(
+ autotune,
+ flashinfer_autotune_cache_path=lambda _: cache_path,
+ get_flashinfer_autotune_skip_ops=lambda _: skip_ops,
+ get_exec=lambda: SimpleNamespace(
+ moe=SimpleNamespace(moe_runner_backend="flashinfer_megamoe")
+ ),
+ get_eager_max_batch_size=lambda bs: bs,
+ max_speculative_num_draft_tokens=lambda: width,
+ max_prefill_buffer_tokens=lambda: max(chunk_size, 0),
+ get_schedule=lambda: SimpleNamespace(max_prefill_tokens=16384),
+ ),
+ patch.object(
+ autotune.torch.cuda,
+ "current_stream",
+ return_value=Mock(),
+ ),
+ patch.object(
+ autotune.torch,
+ "get_device_module",
+ return_value=SimpleNamespace(stream=lambda _: nullcontext()),
+ ),
+ ):
+ general.reset_mock()
+ with autotune.flashinfer_autotune_context(
+ runner, run_lm_head=False
+ ):
+ general.assert_called_once_with(
+ True,
+ cache=None,
+ skip_ops=skip_ops,
+ )
+ context = mega_autotune._active_context
+ if prefill is None:
+ self.assertIs(context, previous)
+ else:
+ self.assertIsNot(context, previous)
+ runner.max_decode_logits_rows.assert_called_with()
+ runner.decode_num_tokens_per_req.assert_called_with(
+ num_draft_tokens=width
+ )
+ self.assertEqual(context.decode_num_tokens, decode)
+ self.assertEqual(context.prefill_num_tokens, prefill)
+ self.assertIs(mega_autotune._active_context, previous)
+
+ # Cache-disabled startups in the same second must still write to
+ # different files, even if they use the same device and rank.
+ with (
+ get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
+ patch.dict(sys.modules, modules),
+ patch.dict(os.environ, SGLANG_FLASHINFER_AUTOTUNE_CACHE="0"),
+ patch.multiple(
+ autotune,
+ flashinfer_autotune_cache_path=lambda _: cache_path,
+ get_flashinfer_autotune_skip_ops=lambda _: set(),
+ get_exec=lambda: SimpleNamespace(
+ moe=SimpleNamespace(moe_runner_backend="triton")
+ ),
+ ),
+ patch.object(autotune.datetime, "datetime") as clock,
+ patch.object(
+ autotune.torch.cuda, "current_stream", return_value=Mock()
+ ),
+ patch.object(
+ autotune.torch,
+ "get_device_module",
+ return_value=SimpleNamespace(stream=lambda _: nullcontext()),
+ ),
+ ):
+ clock.now.return_value.strftime.return_value = "20260922_010000"
+ paths = []
+ for _ in range(2):
+ with autotune.flashinfer_autotune_context(
+ runner, run_lm_head=False
+ ):
+ paths.append(Path(general.call_args.kwargs["cache"]))
+ self.assertNotEqual(*paths)
+ for path in paths:
+ self.assertEqual(path.parent, cache_path.parent / "runs")
+ self.assertTrue(path.name.startswith("rank.20260922_010000."))
+
+
class TestAutotuneCacheDigest(CustomTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
@@ -249,6 +390,13 @@ class TestAutotuneCachePhases(CustomTestCase):
patch.object(
autotune, "get_flashinfer_autotune_skip_ops", return_value=set()
),
+ patch.object(
+ autotune,
+ "get_exec",
+ return_value=SimpleNamespace(
+ moe=SimpleNamespace(moe_runner_backend="flashinfer_cutedsl")
+ ),
+ ),
autotune.envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.override(True),
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
):