diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index b7aa1e550b..7bf0292142 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -164,7 +164,7 @@ SGLang supports various environment variables that can be used to configure its `SGLANG_FLASHINFER_AUTOTUNE_CACHE` - Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs. Set to `0` to force re-autotuning on every startup; the fresh result is written to a `runs/..json` sibling file (the canonical cache is left untouched). + Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs, including MegaMoE capacity profiles. Set to `0` to disable disk-cache reuse; the fresh result is written to a `runs/...json` sibling file (the canonical cache is left untouched). Target MegaMoE rank files include the physical GPU UUID. Use separate `SGLANG_CACHE_DIR` roots for engines sharing physical GPUs; a running process retains tactics for already-pooled workspaces. `true` @@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE - Use the CuTe DSL MegaMOE in-kernel FC2 reduction. This can reduce workspace size and improve large-batch performance, but BF16 atomic accumulation is nondeterministic. It is incompatible with quantized combine dtypes. + Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine. "false" diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d4cc184b30..a6a38c32b1 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1097,9 +1097,10 @@ class Envs: # (derived from cuda_graph_max_bs / chunked_prefill_size when unset). SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0) # Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the - # cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging - # region and can win at large batch, but makes the output accumulation order - # nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility. + # cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction; + # FlashInfer's tuning profile may still select external reduction. + # BF16 atomic accumulation is nondeterministic -- keep off for + # bit-reproducibility. # No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob. SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False) # Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe.py b/python/sglang/srt/layers/moe/flashinfer_megamoe.py index 1e336f5768..1726338398 100644 --- a/python/sglang/srt/layers/moe/flashinfer_megamoe.py +++ b/python/sglang/srt/layers/moe/flashinfer_megamoe.py @@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor: def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]: - import inspect - - if "return_workspace_view" in inspect.signature(mega.forward).parameters: + if mega.supports_output_view: return _forward_megamoe_with_workspace_view return _forward_megamoe_legacy @@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer( layer.hidden_size, ) + process_group = None + if world_size > 1: + from sglang.srt.runtime_context import get_parallel + + process_group = get_parallel().moe_ep_group.device_group + bootstrap = BootstrapConfig( + world_size=world_size, + rank=rank, + device=torch.cuda.current_device(), + process_group=process_group, + ) + fleet_params = FleetParams( + num_experts=layer.num_experts, + max_tokens_per_rank=max_tokens_per_rank, + token_hidden_size=layer.hidden_size, + ) + backend = MegaConfig( + megakernel=megakernel_config, + preprocess_weights=False, + transformed_weights=transformed_weights, + ) mega = MoEEpMegaLayer( - bootstrap=BootstrapConfig( - world_size=world_size, rank=rank, device=torch.cuda.current_device() - ), - fleet_params=FleetParams( - num_experts=layer.num_experts, - max_tokens_per_rank=max_tokens_per_rank, - token_hidden_size=layer.hidden_size, - ), + bootstrap=bootstrap, + fleet_params=fleet_params, # weights already preprocessed in prepare_*; with transformed_weights set # the kernel never reads `weights` (see MoEEpMegaLayer), so pass None. weights=None, - backend=MegaConfig( - megakernel=megakernel_config, - preprocess_weights=False, - transformed_weights=transformed_weights, - ), + backend=backend, ) layer._flashinfer_megamoe_layer = mega - layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega) + if hasattr(megakernel_config, "knobs"): + from sglang.srt.layers.moe.flashinfer_megamoe_autotune import ( + MegaMoeTunedForward, + ) + + layer._flashinfer_megamoe_forward = MegaMoeTunedForward( + bootstrap, fleet_params, backend + ) + else: + layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega) return mega @@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any: top_k=layer.top_k, gate_up_clamp=layer.moe_runner_config.swiglu_limit, apply_topk_in_fc1=True, - in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), + enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), combine_dtype=resolve_flashinfer_megamoe_combine_dtype(), input_norm_const=input_norm_const, - fc1_alpha=layer.g1_alphas, - fc2_alpha=layer.g2_alphas, - fc1_norm_const=layer.w2_input_scale_quant, ), w13_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, @@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any: top_k=layer.top_k, kind="mxfp8_e4m3", gate_up_clamp=layer.moe_runner_config.swiglu_limit, - in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), + enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(), ), w13_scale=layer.w13_weight_scale_inv, w2_scale=layer.w2_weight_scale_inv, @@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe( ) -def _ensure_shared_workspace(mega: Any) -> None: - """Share this layer's workspace across MegaMOE layers with identical - fleet/kernel geometry. - FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const - tensor identity, but sglang binds distinct tensor objects per layer, so - its pool never hits across layers; key on geometry instead, since those - values are re-staged into the workspace per forward rather than baked - into the compiled kernel. Creation is collective, so it must only happen - on a layer's first forward, under warmup's cross-rank lockstep. - """ - if mega._workspace is not None: - return - fp = mega._fleet_params - kc = mega._megakernel_config - mc = mega._mega_config - from sglang.srt.runtime_context import get_resources - - key = ( - getattr(kc, "kernel_name", kc.__class__.__name__), - mega._bootstrap.world_size, - fp.num_experts, - fp.max_tokens_per_rank, - fp.token_hidden_size, - kc.top_k, - kc.intermediate_size, - getattr(kc, "gate_up_clamp", None), - getattr(kc, "activation_clamp", None), - getattr(kc, "apply_topk_in_fc1", None), - getattr(kc, "kind", None), - getattr(kc, "in_kernel_fc2_reduce", None), - getattr(kc, "combine_dtype", None), - getattr(kc, "token_back_by_dispatch", None), - getattr(kc, "fast_math", None), - mc.quantize_input, - ) - workspaces = get_resources().flashinfer_megamoe_workspaces - shared = workspaces.get(key) - if shared is None: - workspaces[key] = mega._ensure_workspace() - else: - mega._workspace = shared - - @register_fused_func("flashinfer_megamoe", "flashinfer_megamoe") def run_flashinfer_megamoe( dispatch_output: DispatchOutput, @@ -599,7 +571,6 @@ def run_flashinfer_megamoe( topk_weights = topk_output.topk_weights topk_ids = topk_output.topk_ids mega = quant_info.mega - _ensure_shared_workspace(mega) t = MoEEpTensors( hidden_states=x.to(torch.bfloat16), @@ -607,6 +578,7 @@ def run_flashinfer_megamoe( # directly into its final int64 workspace buffer. Keep this path copy-free. topk_ids=topk_ids, topk_weights=topk_weights.to(torch.float32), + # Stage per-layer scales on every call so FlashInfer can pool workspaces. fc1_alpha=quant_info.fc1_alpha, fc2_alpha=quant_info.fc2_alpha, fc1_norm_const=quant_info.fc1_norm_const, diff --git a/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py b/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py new file mode 100644 index 0000000000..19fb1eebf2 --- /dev/null +++ b/python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py @@ -0,0 +1,366 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE.""" + +from __future__ import annotations + +import json +import logging +import os +from bisect import bisect_left +from contextlib import contextmanager +from dataclasses import dataclass, field, fields, replace +from functools import cached_property +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +import torch + +logger = logging.getLogger(__name__) + + +@dataclass +class _AutotuneContext: + decode_num_tokens: int + prefill_num_tokens: int + directory: Path + records: dict + staged_profiles: dict[str, Path] = field(default_factory=dict) + dirty: bool = False + + +_active_context: _AutotuneContext | None = None +# Native workspace pooling spans startup contexts. Keep their +# selected tactics consistent too; a cache-path change must not retune storage +# already captured by another layer. These payloads own no weights or tensors. +_session_winners: dict[tuple[Any, int, str], dict] = {} +_CACHE_NAMESPACE = "sglang_flashinfer_megamoe" + + +@contextmanager +def megamoe_autotune_context( + decode_num_tokens: int, + prefill_num_tokens: int, +): + """Prepare profiles inside the engine's pre-capture FlashInfer autotune pass.""" + from flashinfer.autotuner import AutoTuner + + global _active_context + tuner = AutoTuner.get() + previous = _active_context + with TemporaryDirectory(prefix="sglang-megamoe-") as directory: + context = _AutotuneContext( + decode_num_tokens, + prefill_num_tokens, + Path(directory), + tuner.get_namespaced_records(_CACHE_NAMESPACE), + ) + _active_context = context + try: + yield + finally: + _active_context = previous + if context.dirty: + tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records) + + +@contextmanager +def _native_cache(path: Path): + # Native AUTO exposes its winner through this file; generic autotune owns + # persistence. Each geometry is staged once per startup context. + name = "FLASHINFER_MOE_EP_KNOB_CACHE" + previous = os.environ.get(name) + os.environ[name] = str(path) + try: + yield + finally: + if previous is None: + os.environ.pop(name, None) + else: + os.environ[name] = previous + + +def _capacities(limit: int) -> list[int]: + limit = max(1, limit) + return sorted({1 << i for i in range(limit.bit_length())} | {limit}) + + +def _profile_inputs(tensors, capacity: int, num_experts: int): + count = tensors.num_tokens + if count: + + def resize(tensor): + if capacity <= count: + return tensor[:capacity] + return tensor.repeat( + ((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1) + )[:capacity] + + hidden = resize(tensors.hidden_states) + ids = resize(tensors.topk_ids) + scores = resize(tensors.topk_weights) + scales = resize(tensors.scales) if tensors.scales is not None else None + else: + # Idle ranks still enter every collective tuning iteration. + hidden = tensors.hidden_states.new_zeros( + (capacity, tensors.hidden_states.shape[1]) + ) + top_k = tensors.topk_ids.shape[1] + ids = ( + torch.arange( + capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device + ).reshape(capacity, top_k) + % num_experts + ) + scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k) + scales = ( + tensors.scales.new_ones((capacity, *tensors.scales.shape[1:])) + if tensors.scales is not None + else None + ) + return replace( + tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales + ) + + +def _valid_native_cache(payload) -> bool: + if not isinstance(payload, dict) or payload.get("version") != 1: + return False + entries = payload.get("entries") + return ( + isinstance(entries, list) + and len(entries) == 1 + and isinstance(entries[0], dict) + and isinstance(entries[0].get("knobs"), dict) + and bool(entries[0]["knobs"]) + ) + + +class MegaMoeTunedForward: + """Own public workspace handles; FlashInfer pools their storage across layers. + + The caller installs this for backends with prepared weights and default + ``knobs=None``. Preparation must precede ordinary forwarding and capture. + """ + + def __init__(self, bootstrap, fleet_params, backend): + if backend.megakernel.knobs is not None: + raise ValueError("MegaMoE startup tuning requires knobs=None") + if backend.preprocess_weights or backend.transformed_weights is None: + raise ValueError("MegaMoE startup tuning requires prepared weights") + self.bootstrap = bootstrap + self.fleet_params = fleet_params + self.backend = backend + self.workspaces: dict[int, Any] = {} + self._fallback_used = False + self._profile_capacities: tuple[int, ...] = () + + def _common_tokens(self, tensors) -> int: + if self.bootstrap.world_size == 1: + return tensors.num_tokens + from sglang.srt.layers.dp_attention import get_dp_global_num_tokens + + counts = get_dp_global_num_tokens() + if counts is None: + raise RuntimeError( + "MegaMoE workspace selection requires synchronized DP token counts" + ) + return max(counts, default=0) + + @cached_property + def _geometry_key(self) -> str: + config = {} + for item in fields(self.backend.megakernel): + # Per-layer scale values do not change the pooled kernel geometry. + if item.name in ( + "knobs", + "input_norm_const", + "fc1_norm_const", + "fc1_alpha", + "fc2_alpha", + ): + continue + value = getattr(self.backend.megakernel, item.name) + if not isinstance(value, torch.Tensor): + config[item.name] = value + # The enclosing flashinfer.autotune context validates runtime/compiler + # metadata. This key only adds the MegaMoE configuration and geometry. + return json.dumps( + { + "torch": torch.__version__, + "mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"), + "config": config, + "world_size": self.bootstrap.world_size, + "hidden": self.fleet_params.token_hidden_size, + "num_experts": self.fleet_params.num_experts, + }, + sort_keys=True, + ) + + def _require_all_ranks(self, success: bool, message: str) -> None: + if self.bootstrap.world_size > 1: + import torch.distributed as dist + + ready = torch.tensor(int(success), dtype=torch.int32, device="cuda") + dist.all_reduce( + ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group + ) + success = bool(ready.item()) + if not success: + raise RuntimeError(message) + + def _broadcast(self, value): + if self.bootstrap.world_size == 1: + return value + import torch.distributed as dist + + group = self.bootstrap.process_group + values = [value] + dist.broadcast_object_list(values, group=group, group_src=0) + return values[0] + + def _tune(self, tensors, capacity: int, native_path: Path): + from flashinfer.moe_ep import MoEEpMegaLayer + + backend = replace( + self.backend, megakernel=replace(self.backend.megakernel, knobs="auto") + ) + temporary = MoEEpMegaLayer( + bootstrap=self.bootstrap, + fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity), + weights=None, + backend=backend, + ) + payload = None + try: + temporary.warmup(tensors) + if self.bootstrap.rank == 0: + try: + payload = json.loads(native_path.read_text()) + except (OSError, ValueError): + pass + finally: + temporary.destroy() + payload = self._broadcast(payload) + if not _valid_native_cache(payload): + raise RuntimeError("MegaMoE autotune did not record its selected tactic") + return payload + + def _prepare_profile(self, mega, tensors, capacity: int) -> None: + context = _active_context + assert context is not None + profile_key = f"{self._geometry_key}:{capacity}" + profile_inputs = _profile_inputs( + tensors, capacity, self.fleet_params.num_experts + ) + native_path = context.staged_profiles.get(profile_key) + needs_staging = native_path is None + if needs_staging: + native_path = ( + context.directory / f"profile{len(context.staged_profiles)}.json" + ) + with _native_cache(native_path): + if needs_staging: + key = ( + self.bootstrap.process_group, + torch.cuda.current_device(), + profile_key, + ) + payload = _session_winners.get(key) + if payload is None: + payload = self._broadcast( + context.records.get(profile_key) + if self.bootstrap.rank == 0 + else None + ) + source = "cache hit" + if not _valid_native_cache(payload): + payload = self._tune(profile_inputs, capacity, native_path) + source = "tuned" + _session_winners[key] = payload + if self.bootstrap.rank == 0: + logger.info( + "MegaMoE %s: capacity=%s, knobs=%s", + source, + capacity, + payload["entries"][0]["knobs"], + ) + # The backend owns dtype/layout keys and loads its original payload. + try: + native_path.write_text(json.dumps(payload)) + written = True + except OSError: + written = False + self._require_all_ranks( + written, + "Cannot stage MegaMoE's selected tactic for workspace creation", + ) + context.staged_profiles[profile_key] = native_path + if context.records.get(profile_key) != payload: + context.records[profile_key] = payload + context.dirty = True + workspace = mega.create_workspace(capacity) + mega.warmup(profile_inputs, workspace=workspace) + self.workspaces[capacity] = workspace + + @staticmethod + def _forward(mega, tensors, workspace=None): + if mega.supports_output_view: + return mega.forward( + tensors, workspace=workspace, return_workspace_view=True + ) + return mega.forward(tensors, workspace=workspace) + + def __call__(self, mega, tensors): + context = _active_context + if context is None and not self.workspaces: + self._fallback_used = True + return self._forward(mega, tensors) + if context is not None and not self.workspaces: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "MegaMoE autotune must complete before CUDA graph capture" + ) + if self._fallback_used: + raise RuntimeError( + "MegaMoE autotune must precede its first ordinary forward" + ) + import torch.distributed as dist + + if ( + dist.is_initialized() + and self.bootstrap.world_size != dist.get_world_size() + ): + raise RuntimeError( + "FlashInfer MegaMoE startup tuning currently requires the full WORLD process group" + ) + # The startup dummy can be much larger than captured decode batches. + # Keep small-N profiles bounded by decode, plus one prefill profile. + capacities = set(_capacities(context.decode_num_tokens)) + capacities.add(context.prefill_num_tokens) + for capacity in sorted(capacities): + self._prepare_profile(mega, tensors, capacity) + self._profile_capacities = tuple(sorted(self.workspaces)) + + count = self._common_tokens(tensors) + index = bisect_left(self._profile_capacities, count) + if index < len(self._profile_capacities): + return self._forward( + mega, tensors, self.workspaces[self._profile_capacities[index]] + ) + raise ValueError( + f"MegaMoE received {count} tokens per rank, exceeding its prepared " + f"decode/prefill capacity {max(self.workspaces)}" + ) diff --git a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py index 794fdaffd1..c8d2f35519 100644 --- a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py +++ b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py @@ -19,6 +19,7 @@ import functools import hashlib import json import logging +import uuid from pathlib import Path from typing import TYPE_CHECKING, Callable, Optional @@ -34,8 +35,10 @@ from sglang.srt.runtime_context import ( get_schedule, get_spec, max_prefill_buffer_tokens, + max_speculative_num_draft_tokens, ) from sglang.srt.utils import empty_context, log_info_on_rank0 +from sglang.srt.utils.common import get_eager_max_batch_size if TYPE_CHECKING: from sglang.srt.distributed.parallel_state import GroupCoordinator @@ -94,6 +97,7 @@ def should_run_flashinfer_autotune( "flashinfer_mxfp4", "flashinfer_cutedsl", "flashinfer_cutlass", + "flashinfer_megamoe", ] from sglang.srt.layers.quantization.fp4_utils import ( @@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path: / cache_key ) cache_dir.mkdir(parents=True, exist_ok=True) - return ( - cache_dir - / f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json" - ) + rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}" + if ( + get_exec().moe.moe_runner_backend == "flashinfer_megamoe" + and not mr.is_draft_worker + ): + from sglang.srt.platforms import current_platform + + # Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's + # namespaced records currently require one writer per file. Follow the + # P2P cache's device isolation, using physical UUIDs for stable identity + # across CUDA index remapping and restarts. Engines sharing physical GPUs + # must use separate SGLANG_CACHE_DIRs. + rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}" + return cache_dir / f"{rank_key}.json" def _autotune_tactic_sync_group( @@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") runs_dir = cache_path.parent / "runs" runs_dir.mkdir(parents=True, exist_ok=True) - autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}" + autotune_cache = runs_dir / ( + f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}" + ) logger.info( "Running FlashInfer autotune (cache reuse DISABLED via " "SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s", @@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) mr.forward_stream.wait_stream(torch.cuda.current_stream()) with torch.get_device_module(mr.device).stream(mr.forward_stream): from sglang.srt.layers.logits_processor import autotune_dummy_run_mode + from sglang.srt.layers.moe.flashinfer_megamoe_autotune import ( + megamoe_autotune_context, + ) skip_ops = get_flashinfer_autotune_skip_ops(mr) # autotune(cache=...) clears all file-loaded tactics on entry, which would drop @@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) tuner = AutoTuner.get() if reuse_cache and autotune_cache.is_file(): tuner.load_configs(str(autotune_cache)) + mega_context = empty_context() + if ( + get_exec().moe.moe_runner_backend == "flashinfer_megamoe" + and not mr.is_draft_worker + and "flashinfer_megamoe" not in skip_ops + ): + # The chunk size is already per DP rank. Prepare one prefill profile + # independently of the optional full-model EXTEND autotune pass. + prefill_num_tokens = ( + max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens + ) + mega_context = megamoe_autotune_context( + decode_num_tokens=max( + mr.max_decode_logits_rows(), + get_eager_max_batch_size(mr.max_running_requests) + * mr.decode_num_tokens_per_req( + num_draft_tokens=max_speculative_num_draft_tokens() + ), + ), + prefill_num_tokens=prefill_num_tokens, + ) with ( _autotune_process_group(sync_group), autotune( @@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) cache=None if reuse_cache else str(autotune_cache), skip_ops=skip_ops, ), + mega_context, autotune_dummy_run_mode(run_lm_head=run_lm_head), ): yield diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index bcea02a32d..0e64489406 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -644,7 +644,6 @@ class Resources(_FlagGroupBase): # Persistent reusable CUDA events for non-EP DP TBO, keyed by # (kind, subbatch) — see dp_attention._tbo_event for why reuse matters. tbo_event_pool: dict = msgspec.field(default_factory=dict) - flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict) # State capturers (installed by their subsystems when capture is on). indexer_capturer: Any = None experts_capturer: Any = None diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index fa07444af5..7ac94f1ed7 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None): # reuse this flag's DP-sync bookkeeping (uniform global_num_tokens + # max-based graph bucket). See #30432 re: the misleading flag name. return True + elif moe_a2a_backend.is_flashinfer_megamoe(): + # MegaMoE's capacity profiles require the same decode graph + # bucket on every EP rank. As for FlashInfer A2A above, this only + # enables DP-count bookkeeping; expert inputs remain local. + return True elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get(): # Same bookkeeping, for the same reason. Bounding mori's receive # buffer means baking a fan-in size into a captured graph, and the diff --git a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py index cd73104ed9..640125fc18 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_megamoe.py +++ b/test/registered/unit/layers/moe/test_flashinfer_megamoe.py @@ -3,6 +3,7 @@ import sys import types from pathlib import Path +import pytest import torch from sglang.test.ci.ci_register import register_cpu_ci @@ -69,6 +70,7 @@ def _load_megamoe_module(monkeypatch): for name, module in fake_modules.items(): monkeypatch.setitem(sys.modules, name, module) + _load_autotune_module(monkeypatch) module_path = ( Path(__file__).resolve().parents[5] / "python/sglang/srt/layers/moe/flashinfer_megamoe.py" @@ -82,6 +84,269 @@ def _load_megamoe_module(monkeypatch): return module +def _load_autotune_module(monkeypatch): + name = "sglang.srt.layers.moe.flashinfer_megamoe_autotune" + path = ( + Path(__file__).resolve().parents[5] + / "python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py" + ) + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + return module + + +def _profile_fixture(monkeypatch, world_size=1, num_tokens=3): + from dataclasses import dataclass + + module = _load_autotune_module(monkeypatch) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + tuner = types.SimpleNamespace( + get_namespaced_records=lambda _: {}, + publish_namespaced_records=lambda *_: None, + ) + monkeypatch.setitem( + sys.modules, + "flashinfer.autotuner", + types.SimpleNamespace(AutoTuner=types.SimpleNamespace(get=lambda: tuner)), + ) + + @dataclass + class Tensors: + hidden_states: torch.Tensor + topk_ids: torch.Tensor + topk_weights: torch.Tensor + fc1_alpha: torch.Tensor + scales: torch.Tensor | None = None + + @property + def num_tokens(self): + return self.hidden_states.shape[0] + + @dataclass + class Config: + knobs: str | None = None + kernel_name: str = "test_backend" + input_norm_const: float = 1.0 + + @dataclass + class Fleet: + max_tokens_per_rank: int = 4 + num_experts: int = 4 + token_hidden_size: int = 4 + + @dataclass + class Backend: + megakernel: Config + transformed_weights: object + preprocess_weights: bool = False + + tensors = Tensors( + torch.ones(num_tokens, 4, dtype=torch.bfloat16), + torch.arange(num_tokens * 2, dtype=torch.int32).reshape(num_tokens, 2) % 4, + torch.full((num_tokens, 2), 0.5, dtype=torch.float32), + torch.ones(4, dtype=torch.float32), + ) + forward = module.MegaMoeTunedForward( + types.SimpleNamespace(world_size=world_size, rank=0, process_group=None), + Fleet(), + Backend(Config(), object()), + ) + calls = [] + + class Mega: + supports_output_view = False + + def forward(self, inputs, **kwargs): + calls.append((inputs, kwargs.get("workspace"))) + return inputs.hidden_states + + return module, forward, tensors, Mega(), calls + + +@pytest.mark.parametrize( + "decode_tokens,expected", [(3, [1, 2, 3, 128]), (8, [1, 2, 4, 8, 128])] +) +def test_megamoe_startup_profiles_are_prepared_once( + monkeypatch, decode_tokens, expected +): + module, forward, tensors, mega, calls = _profile_fixture(monkeypatch, num_tokens=96) + prepared = [] + + def prepare(layer, inputs, capacity): + prepared.append(capacity) + forward.workspaces[capacity] = object() + + monkeypatch.setattr(forward, "_prepare_profile", prepare) + with module.megamoe_autotune_context(decode_tokens, prefill_num_tokens=128): + assert forward(mega, tensors) is tensors.hidden_states + # Bounds may exceed the bootstrap fleet. A large initial dummy must not + # create intermediate prefill profiles. + assert prepared == expected + assert calls[-1][1] is forward.workspaces[128] + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + for count, selected in ((2, 2), (decode_tokens, decode_tokens), (32, 128)): + inputs = module._profile_inputs(tensors, count, num_experts=4) + forward(mega, inputs) + assert prepared == expected + assert calls[-1][1] is forward.workspaces[selected] + with pytest.raises(ValueError, match="exceeding its prepared"): + forward(mega, module._profile_inputs(tensors, 129, num_experts=4)) + + +@pytest.mark.parametrize("local_tokens", [0, 3]) +def test_megamoe_profile_selection_uses_all_dp_ranks(monkeypatch, local_tokens): + module, forward, tensors, mega, calls = _profile_fixture( + monkeypatch, world_size=4, num_tokens=local_tokens + ) + dp = types.ModuleType("sglang.srt.layers.dp_attention") + dp.get_dp_global_num_tokens = lambda: [0, 1, 3, 7] + monkeypatch.setitem(sys.modules, dp.__name__, dp) + forward.workspaces = {1: object(), 4: object(), 8: object(), 128: object()} + forward._profile_capacities = tuple(forward.workspaces) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + forward(mega, tensors) + assert calls[-1][1] is forward.workspaces[8] + + +def test_megamoe_without_tuning_preserves_capture_forward(monkeypatch): + module, forward, tensors, mega, calls = _profile_fixture(monkeypatch) + forward(mega, tensors) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True) + forward(mega, tensors) + assert [workspace for _, workspace in calls] == [None, None] + + +@pytest.mark.parametrize("new_process", [False, True]) +def test_megamoe_profiles_reuse_cached_winner(monkeypatch, new_process): + import json + import os + from dataclasses import replace + from unittest.mock import Mock + + module, first, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=1) + payload = {"version": 1, "entries": [{"knobs": {"native_tactic": 7}}]} + events = [] + staged = [] + native_paths = [] + require_all_ranks = module.MegaMoeTunedForward._require_all_ranks + readiness = Mock() + + def require(self, success, message): + readiness(success) + require_all_ranks(self, success, message) + + monkeypatch.setattr(module.MegaMoeTunedForward, "_require_all_ranks", require) + + class Mega: + supports_output_view = True + + def __init__(self, **kwargs): + self.auto = kwargs.get("backend", first.backend).megakernel.knobs == "auto" + + def warmup(self, inputs, workspace=None): + if self.auto: + events.append("tune") + Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"]).write_text( + json.dumps(payload) + ) + else: + assert workspace is self.workspace + events.append("warmup") + + def destroy(self): + events.append("destroy") + + def create_workspace(self, capacity): + assert capacity == 1 + path = Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"]) + native_paths.append(path) + staged.append(json.loads(path.read_text())) + self.workspace = object() + events.append("create") + return self.workspace + + def forward(self, inputs, *, workspace, return_workspace_view=False): + assert workspace is self.workspace + assert return_workspace_view + return inputs.hidden_states + + fake_moe = types.ModuleType("flashinfer.moe_ep") + fake_moe.MoEEpMegaLayer = Mega + fake_autotuner = types.ModuleType("flashinfer.autotuner") + records = {} + tuner = types.SimpleNamespace( + get_namespaced_records=Mock( + side_effect=lambda namespace: records.get(namespace, {}).copy() + ), + publish_namespaced_records=Mock( + side_effect=lambda namespace, values: records.__setitem__( + namespace, values.copy() + ) + ), + ) + fake_autotuner.AutoTuner = types.SimpleNamespace(get=lambda: tuner) + monkeypatch.setitem(sys.modules, "flashinfer", types.ModuleType("flashinfer")) + monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe) + monkeypatch.setitem(sys.modules, "flashinfer.autotuner", fake_autotuner) + monkeypatch.setenv("FLASHINFER_MOE_EP_KNOB_CACHE", "original-cache.json") + + owners = [] + for phase in range(2): + if owners: + if new_process: + module._session_winners.clear() # Only the loaded cache survives. + else: + records.clear() # A new cache context must preserve pooled choices. + with module.megamoe_autotune_context( + decode_num_tokens=1, + prefill_num_tokens=1, + ): + for scale in (1.0, 2.0): + backend = replace( + first.backend, + megakernel=replace( + first.backend.megakernel, input_norm_const=scale + ), + ) + forward = module.MegaMoeTunedForward( + first.bootstrap, first.fleet_params, backend + ) + mega = Mega() + assert forward(mega, tensors) is tensors.hidden_states + owners.append(mega) + assert readiness.call_count == phase + 1 + assert native_paths[-2] == native_paths[-1] + assert native_paths[-1].is_file() + assert not native_paths[-1].exists() + assert os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"] == "original-cache.json" + assert list(records["sglang_flashinfer_megamoe"].values()) == [payload] + assert tuner.get_namespaced_records.call_count == phase + 1 + + assert tuner.publish_namespaced_records.call_count == (1 if new_process else 2) + assert events == ["tune", "destroy"] + ["create", "warmup"] * 4 + assert staged == [payload] * 4 + assert len({id(owner.workspace) for owner in owners}) == 4 + + +@pytest.mark.parametrize("tokens,capacity", [(0, 8), (3, 8), (3, 1)]) +def test_megamoe_profile_inputs_retain_runtime_scales(monkeypatch, tokens, capacity): + module, _, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=tokens) + tensors.scales = torch.ones(tokens, 2) + profile = module._profile_inputs(tensors, capacity, num_experts=4) + assert profile.hidden_states.shape == (capacity, 4) + assert profile.topk_ids.shape == profile.topk_weights.shape == (capacity, 2) + assert profile.hidden_states.dtype == torch.bfloat16 + assert profile.topk_ids.dtype == torch.int32 + assert profile.topk_weights.dtype == torch.float32 + assert profile.fc1_alpha is tensors.fc1_alpha + assert profile.scales.shape == (capacity, 2) + assert torch.all(profile.scales == 1) + assert torch.all((profile.topk_ids >= 0) & (profile.topk_ids < 4)) + + def test_max_tokens_uses_runtime_context_accessor(monkeypatch): module = _load_megamoe_module(monkeypatch) @@ -112,7 +377,7 @@ def test_adapter_keeps_router_ids_int32(monkeypatch): output = torch.randn_like(hidden_states) class Mega: - _workspace = object() + supports_output_view = False def forward(self, tensors): self.tensors = tensors @@ -140,7 +405,8 @@ def test_adapter_keeps_router_ids_int32(monkeypatch): assert result.hidden_states is output -def test_adapter_requests_workspace_output_view(monkeypatch): +@pytest.mark.parametrize("supports_output_view", [False, True]) +def test_adapter_requests_workspace_output_view(monkeypatch, supports_output_view): module = _load_megamoe_module(monkeypatch) class FakeMoEEpTensors: @@ -160,15 +426,13 @@ def test_adapter_requests_workspace_output_view(monkeypatch): output = torch.randn_like(hidden_states) class Mega: - supports_output_view = True - _workspace = object() - def forward(self, tensors, *, return_workspace_view=False): self.tensors = tensors self.return_workspace_view = return_workspace_view return output mega = Mega() + mega.supports_output_view = supports_output_view dispatch_output = types.SimpleNamespace( hidden_states=hidden_states, topk_output=types.SimpleNamespace( @@ -186,7 +450,7 @@ def test_adapter_requests_workspace_output_view(monkeypatch): assert result.hidden_states is output assert mega.tensors.topk_ids.data_ptr() == topk_ids.data_ptr() assert mega.tensors.topk_ids.dtype == torch.int32 - assert mega.return_workspace_view is True + assert mega.return_workspace_view is supports_output_view def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch): @@ -215,6 +479,4 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch): if __name__ == "__main__": - import pytest - sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py index c8151e4455..14ed53a32a 100644 --- a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py +++ b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py @@ -14,6 +14,7 @@ register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-l import json import multiprocessing import os +import sys import tempfile import traceback import unittest @@ -65,6 +66,146 @@ class TestAutotuneTacticSyncGroup(CustomTestCase): self.assertIsNone(_autotune_tactic_sync_group(tp_group)) +class TestMegaMoEAutotuneStartup(CustomTestCase): + def test_context_bridges_profile_limits(self): + from sglang.srt.layers.moe import flashinfer_megamoe_autotune as mega_autotune + + runner = SimpleNamespace( + device="cuda", + forward_stream=Mock(), + max_decode_logits_rows=Mock(), + decode_num_tokens_per_req=Mock(), + max_running_requests=256, + is_draft_worker=False, + ) + general = Mock(side_effect=lambda *_args, **_kwargs: nullcontext()) + modules = { + "flashinfer.autotuner": SimpleNamespace( + AutoTuner=SimpleNamespace( + get=lambda: SimpleNamespace( + get_namespaced_records=lambda _: {}, + load_configs=lambda _: None, + save_configs=lambda _: None, + publish_namespaced_records=lambda *_: None, + ) + ), + _collect_metadata=lambda: ENV, + autotune=general, + get_autotune_process_group=lambda: None, + set_autotune_process_group=lambda _: None, + ), + "sglang.srt.layers.logits_processor": SimpleNamespace( + autotune_dummy_run_mode=lambda **_: nullcontext() + ), + } + # The resolved chunk size is already per DP rank. Disabled chunking + # uses the scheduler's prefill ceiling; skip-op suppresses the bridge. + # (chunk, skip, draft, graph rows, maximum MTP width, decode, prefill) + cases = [ + (4096, False, False, 128 * 6, 6, 256 * 6, 4096), + (-1, False, False, 8192, 6, 8192, 16384), + (4096, False, False, 4096, 32, 8192, 4096), + (4096, True, False, 128 * 6, 6, None, None), + (4096, False, True, 128 * 6, 6, None, None), + ] + previous = mega_autotune._active_context + with tempfile.TemporaryDirectory() as directory: + cache_path = Path(directory) / "rank.json" + for chunk_size, skip, draft, graph_rows, width, decode, prefill in cases: + runner.is_draft_worker = draft + runner.max_decode_logits_rows.return_value = graph_rows + runner.decode_num_tokens_per_req.return_value = width + skip_ops = {"flashinfer_megamoe"} if skip else set() + with ( + self.subTest(chunk_size=chunk_size, skip=skip, draft=draft), + get_parallel().override(tp_group=SimpleNamespace(world_size=1)), + patch.dict(sys.modules, modules), + patch.dict( + os.environ, + SGLANG_FLASHINFER_AUTOTUNE_CACHE="1", + ), + patch.multiple( + autotune, + flashinfer_autotune_cache_path=lambda _: cache_path, + get_flashinfer_autotune_skip_ops=lambda _: skip_ops, + get_exec=lambda: SimpleNamespace( + moe=SimpleNamespace(moe_runner_backend="flashinfer_megamoe") + ), + get_eager_max_batch_size=lambda bs: bs, + max_speculative_num_draft_tokens=lambda: width, + max_prefill_buffer_tokens=lambda: max(chunk_size, 0), + get_schedule=lambda: SimpleNamespace(max_prefill_tokens=16384), + ), + patch.object( + autotune.torch.cuda, + "current_stream", + return_value=Mock(), + ), + patch.object( + autotune.torch, + "get_device_module", + return_value=SimpleNamespace(stream=lambda _: nullcontext()), + ), + ): + general.reset_mock() + with autotune.flashinfer_autotune_context( + runner, run_lm_head=False + ): + general.assert_called_once_with( + True, + cache=None, + skip_ops=skip_ops, + ) + context = mega_autotune._active_context + if prefill is None: + self.assertIs(context, previous) + else: + self.assertIsNot(context, previous) + runner.max_decode_logits_rows.assert_called_with() + runner.decode_num_tokens_per_req.assert_called_with( + num_draft_tokens=width + ) + self.assertEqual(context.decode_num_tokens, decode) + self.assertEqual(context.prefill_num_tokens, prefill) + self.assertIs(mega_autotune._active_context, previous) + + # Cache-disabled startups in the same second must still write to + # different files, even if they use the same device and rank. + with ( + get_parallel().override(tp_group=SimpleNamespace(world_size=1)), + patch.dict(sys.modules, modules), + patch.dict(os.environ, SGLANG_FLASHINFER_AUTOTUNE_CACHE="0"), + patch.multiple( + autotune, + flashinfer_autotune_cache_path=lambda _: cache_path, + get_flashinfer_autotune_skip_ops=lambda _: set(), + get_exec=lambda: SimpleNamespace( + moe=SimpleNamespace(moe_runner_backend="triton") + ), + ), + patch.object(autotune.datetime, "datetime") as clock, + patch.object( + autotune.torch.cuda, "current_stream", return_value=Mock() + ), + patch.object( + autotune.torch, + "get_device_module", + return_value=SimpleNamespace(stream=lambda _: nullcontext()), + ), + ): + clock.now.return_value.strftime.return_value = "20260922_010000" + paths = [] + for _ in range(2): + with autotune.flashinfer_autotune_context( + runner, run_lm_head=False + ): + paths.append(Path(general.call_args.kwargs["cache"])) + self.assertNotEqual(*paths) + for path in paths: + self.assertEqual(path.parent, cache_path.parent / "runs") + self.assertTrue(path.name.startswith("rank.20260922_010000.")) + + class TestAutotuneCacheDigest(CustomTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() @@ -249,6 +390,13 @@ class TestAutotuneCachePhases(CustomTestCase): patch.object( autotune, "get_flashinfer_autotune_skip_ops", return_value=set() ), + patch.object( + autotune, + "get_exec", + return_value=SimpleNamespace( + moe=SimpleNamespace(moe_runner_backend="flashinfer_cutedsl") + ), + ), autotune.envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.override(True), get_parallel().override(tp_group=SimpleNamespace(world_size=1)), ):