[FlashInfer] Add startup autotuning for MegaMoE (#43789)
This commit is contained in:
@@ -1097,9 +1097,10 @@ class Envs:
|
||||
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
|
||||
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
|
||||
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
|
||||
# cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging
|
||||
# region and can win at large batch, but makes the output accumulation order
|
||||
# nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility.
|
||||
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
|
||||
# FlashInfer's tuning profile may still select external reduction.
|
||||
# BF16 atomic accumulation is nondeterministic -- keep off for
|
||||
# bit-reproducibility.
|
||||
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
|
||||
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False)
|
||||
# Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE
|
||||
|
||||
@@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor:
|
||||
|
||||
|
||||
def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]:
|
||||
import inspect
|
||||
|
||||
if "return_workspace_view" in inspect.signature(mega.forward).parameters:
|
||||
if mega.supports_output_view:
|
||||
return _forward_megamoe_with_workspace_view
|
||||
return _forward_megamoe_legacy
|
||||
|
||||
@@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer(
|
||||
layer.hidden_size,
|
||||
)
|
||||
|
||||
process_group = None
|
||||
if world_size > 1:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
process_group = get_parallel().moe_ep_group.device_group
|
||||
bootstrap = BootstrapConfig(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
device=torch.cuda.current_device(),
|
||||
process_group=process_group,
|
||||
)
|
||||
fleet_params = FleetParams(
|
||||
num_experts=layer.num_experts,
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
token_hidden_size=layer.hidden_size,
|
||||
)
|
||||
backend = MegaConfig(
|
||||
megakernel=megakernel_config,
|
||||
preprocess_weights=False,
|
||||
transformed_weights=transformed_weights,
|
||||
)
|
||||
mega = MoEEpMegaLayer(
|
||||
bootstrap=BootstrapConfig(
|
||||
world_size=world_size, rank=rank, device=torch.cuda.current_device()
|
||||
),
|
||||
fleet_params=FleetParams(
|
||||
num_experts=layer.num_experts,
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
token_hidden_size=layer.hidden_size,
|
||||
),
|
||||
bootstrap=bootstrap,
|
||||
fleet_params=fleet_params,
|
||||
# weights already preprocessed in prepare_*; with transformed_weights set
|
||||
# the kernel never reads `weights` (see MoEEpMegaLayer), so pass None.
|
||||
weights=None,
|
||||
backend=MegaConfig(
|
||||
megakernel=megakernel_config,
|
||||
preprocess_weights=False,
|
||||
transformed_weights=transformed_weights,
|
||||
),
|
||||
backend=backend,
|
||||
)
|
||||
layer._flashinfer_megamoe_layer = mega
|
||||
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
|
||||
if hasattr(megakernel_config, "knobs"):
|
||||
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
|
||||
MegaMoeTunedForward,
|
||||
)
|
||||
|
||||
layer._flashinfer_megamoe_forward = MegaMoeTunedForward(
|
||||
bootstrap, fleet_params, backend
|
||||
)
|
||||
else:
|
||||
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
|
||||
return mega
|
||||
|
||||
|
||||
@@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
top_k=layer.top_k,
|
||||
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
|
||||
apply_topk_in_fc1=True,
|
||||
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
|
||||
input_norm_const=input_norm_const,
|
||||
fc1_alpha=layer.g1_alphas,
|
||||
fc2_alpha=layer.g2_alphas,
|
||||
fc1_norm_const=layer.w2_input_scale_quant,
|
||||
),
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
@@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
top_k=layer.top_k,
|
||||
kind="mxfp8_e4m3",
|
||||
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
|
||||
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
),
|
||||
w13_scale=layer.w13_weight_scale_inv,
|
||||
w2_scale=layer.w2_weight_scale_inv,
|
||||
@@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
|
||||
)
|
||||
|
||||
|
||||
def _ensure_shared_workspace(mega: Any) -> None:
|
||||
"""Share this layer's workspace across MegaMOE layers with identical
|
||||
fleet/kernel geometry.
|
||||
FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const
|
||||
tensor identity, but sglang binds distinct tensor objects per layer, so
|
||||
its pool never hits across layers; key on geometry instead, since those
|
||||
values are re-staged into the workspace per forward rather than baked
|
||||
into the compiled kernel. Creation is collective, so it must only happen
|
||||
on a layer's first forward, under warmup's cross-rank lockstep.
|
||||
"""
|
||||
if mega._workspace is not None:
|
||||
return
|
||||
fp = mega._fleet_params
|
||||
kc = mega._megakernel_config
|
||||
mc = mega._mega_config
|
||||
from sglang.srt.runtime_context import get_resources
|
||||
|
||||
key = (
|
||||
getattr(kc, "kernel_name", kc.__class__.__name__),
|
||||
mega._bootstrap.world_size,
|
||||
fp.num_experts,
|
||||
fp.max_tokens_per_rank,
|
||||
fp.token_hidden_size,
|
||||
kc.top_k,
|
||||
kc.intermediate_size,
|
||||
getattr(kc, "gate_up_clamp", None),
|
||||
getattr(kc, "activation_clamp", None),
|
||||
getattr(kc, "apply_topk_in_fc1", None),
|
||||
getattr(kc, "kind", None),
|
||||
getattr(kc, "in_kernel_fc2_reduce", None),
|
||||
getattr(kc, "combine_dtype", None),
|
||||
getattr(kc, "token_back_by_dispatch", None),
|
||||
getattr(kc, "fast_math", None),
|
||||
mc.quantize_input,
|
||||
)
|
||||
workspaces = get_resources().flashinfer_megamoe_workspaces
|
||||
shared = workspaces.get(key)
|
||||
if shared is None:
|
||||
workspaces[key] = mega._ensure_workspace()
|
||||
else:
|
||||
mega._workspace = shared
|
||||
|
||||
|
||||
@register_fused_func("flashinfer_megamoe", "flashinfer_megamoe")
|
||||
def run_flashinfer_megamoe(
|
||||
dispatch_output: DispatchOutput,
|
||||
@@ -599,7 +571,6 @@ def run_flashinfer_megamoe(
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
mega = quant_info.mega
|
||||
_ensure_shared_workspace(mega)
|
||||
|
||||
t = MoEEpTensors(
|
||||
hidden_states=x.to(torch.bfloat16),
|
||||
@@ -607,6 +578,7 @@ def run_flashinfer_megamoe(
|
||||
# directly into its final int64 workspace buffer. Keep this path copy-free.
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights.to(torch.float32),
|
||||
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
|
||||
fc1_alpha=quant_info.fc1_alpha,
|
||||
fc2_alpha=quant_info.fc2_alpha,
|
||||
fc1_norm_const=quant_info.fc1_norm_const,
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from bisect import bisect_left
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _AutotuneContext:
|
||||
decode_num_tokens: int
|
||||
prefill_num_tokens: int
|
||||
directory: Path
|
||||
records: dict
|
||||
staged_profiles: dict[str, Path] = field(default_factory=dict)
|
||||
dirty: bool = False
|
||||
|
||||
|
||||
_active_context: _AutotuneContext | None = None
|
||||
# Native workspace pooling spans startup contexts. Keep their
|
||||
# selected tactics consistent too; a cache-path change must not retune storage
|
||||
# already captured by another layer. These payloads own no weights or tensors.
|
||||
_session_winners: dict[tuple[Any, int, str], dict] = {}
|
||||
_CACHE_NAMESPACE = "sglang_flashinfer_megamoe"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def megamoe_autotune_context(
|
||||
decode_num_tokens: int,
|
||||
prefill_num_tokens: int,
|
||||
):
|
||||
"""Prepare profiles inside the engine's pre-capture FlashInfer autotune pass."""
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
global _active_context
|
||||
tuner = AutoTuner.get()
|
||||
previous = _active_context
|
||||
with TemporaryDirectory(prefix="sglang-megamoe-") as directory:
|
||||
context = _AutotuneContext(
|
||||
decode_num_tokens,
|
||||
prefill_num_tokens,
|
||||
Path(directory),
|
||||
tuner.get_namespaced_records(_CACHE_NAMESPACE),
|
||||
)
|
||||
_active_context = context
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_active_context = previous
|
||||
if context.dirty:
|
||||
tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _native_cache(path: Path):
|
||||
# Native AUTO exposes its winner through this file; generic autotune owns
|
||||
# persistence. Each geometry is staged once per startup context.
|
||||
name = "FLASHINFER_MOE_EP_KNOB_CACHE"
|
||||
previous = os.environ.get(name)
|
||||
os.environ[name] = str(path)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if previous is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = previous
|
||||
|
||||
|
||||
def _capacities(limit: int) -> list[int]:
|
||||
limit = max(1, limit)
|
||||
return sorted({1 << i for i in range(limit.bit_length())} | {limit})
|
||||
|
||||
|
||||
def _profile_inputs(tensors, capacity: int, num_experts: int):
|
||||
count = tensors.num_tokens
|
||||
if count:
|
||||
|
||||
def resize(tensor):
|
||||
if capacity <= count:
|
||||
return tensor[:capacity]
|
||||
return tensor.repeat(
|
||||
((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1)
|
||||
)[:capacity]
|
||||
|
||||
hidden = resize(tensors.hidden_states)
|
||||
ids = resize(tensors.topk_ids)
|
||||
scores = resize(tensors.topk_weights)
|
||||
scales = resize(tensors.scales) if tensors.scales is not None else None
|
||||
else:
|
||||
# Idle ranks still enter every collective tuning iteration.
|
||||
hidden = tensors.hidden_states.new_zeros(
|
||||
(capacity, tensors.hidden_states.shape[1])
|
||||
)
|
||||
top_k = tensors.topk_ids.shape[1]
|
||||
ids = (
|
||||
torch.arange(
|
||||
capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device
|
||||
).reshape(capacity, top_k)
|
||||
% num_experts
|
||||
)
|
||||
scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k)
|
||||
scales = (
|
||||
tensors.scales.new_ones((capacity, *tensors.scales.shape[1:]))
|
||||
if tensors.scales is not None
|
||||
else None
|
||||
)
|
||||
return replace(
|
||||
tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales
|
||||
)
|
||||
|
||||
|
||||
def _valid_native_cache(payload) -> bool:
|
||||
if not isinstance(payload, dict) or payload.get("version") != 1:
|
||||
return False
|
||||
entries = payload.get("entries")
|
||||
return (
|
||||
isinstance(entries, list)
|
||||
and len(entries) == 1
|
||||
and isinstance(entries[0], dict)
|
||||
and isinstance(entries[0].get("knobs"), dict)
|
||||
and bool(entries[0]["knobs"])
|
||||
)
|
||||
|
||||
|
||||
class MegaMoeTunedForward:
|
||||
"""Own public workspace handles; FlashInfer pools their storage across layers.
|
||||
|
||||
The caller installs this for backends with prepared weights and default
|
||||
``knobs=None``. Preparation must precede ordinary forwarding and capture.
|
||||
"""
|
||||
|
||||
def __init__(self, bootstrap, fleet_params, backend):
|
||||
if backend.megakernel.knobs is not None:
|
||||
raise ValueError("MegaMoE startup tuning requires knobs=None")
|
||||
if backend.preprocess_weights or backend.transformed_weights is None:
|
||||
raise ValueError("MegaMoE startup tuning requires prepared weights")
|
||||
self.bootstrap = bootstrap
|
||||
self.fleet_params = fleet_params
|
||||
self.backend = backend
|
||||
self.workspaces: dict[int, Any] = {}
|
||||
self._fallback_used = False
|
||||
self._profile_capacities: tuple[int, ...] = ()
|
||||
|
||||
def _common_tokens(self, tensors) -> int:
|
||||
if self.bootstrap.world_size == 1:
|
||||
return tensors.num_tokens
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
|
||||
counts = get_dp_global_num_tokens()
|
||||
if counts is None:
|
||||
raise RuntimeError(
|
||||
"MegaMoE workspace selection requires synchronized DP token counts"
|
||||
)
|
||||
return max(counts, default=0)
|
||||
|
||||
@cached_property
|
||||
def _geometry_key(self) -> str:
|
||||
config = {}
|
||||
for item in fields(self.backend.megakernel):
|
||||
# Per-layer scale values do not change the pooled kernel geometry.
|
||||
if item.name in (
|
||||
"knobs",
|
||||
"input_norm_const",
|
||||
"fc1_norm_const",
|
||||
"fc1_alpha",
|
||||
"fc2_alpha",
|
||||
):
|
||||
continue
|
||||
value = getattr(self.backend.megakernel, item.name)
|
||||
if not isinstance(value, torch.Tensor):
|
||||
config[item.name] = value
|
||||
# The enclosing flashinfer.autotune context validates runtime/compiler
|
||||
# metadata. This key only adds the MegaMoE configuration and geometry.
|
||||
return json.dumps(
|
||||
{
|
||||
"torch": torch.__version__,
|
||||
"mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"),
|
||||
"config": config,
|
||||
"world_size": self.bootstrap.world_size,
|
||||
"hidden": self.fleet_params.token_hidden_size,
|
||||
"num_experts": self.fleet_params.num_experts,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
def _require_all_ranks(self, success: bool, message: str) -> None:
|
||||
if self.bootstrap.world_size > 1:
|
||||
import torch.distributed as dist
|
||||
|
||||
ready = torch.tensor(int(success), dtype=torch.int32, device="cuda")
|
||||
dist.all_reduce(
|
||||
ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group
|
||||
)
|
||||
success = bool(ready.item())
|
||||
if not success:
|
||||
raise RuntimeError(message)
|
||||
|
||||
def _broadcast(self, value):
|
||||
if self.bootstrap.world_size == 1:
|
||||
return value
|
||||
import torch.distributed as dist
|
||||
|
||||
group = self.bootstrap.process_group
|
||||
values = [value]
|
||||
dist.broadcast_object_list(values, group=group, group_src=0)
|
||||
return values[0]
|
||||
|
||||
def _tune(self, tensors, capacity: int, native_path: Path):
|
||||
from flashinfer.moe_ep import MoEEpMegaLayer
|
||||
|
||||
backend = replace(
|
||||
self.backend, megakernel=replace(self.backend.megakernel, knobs="auto")
|
||||
)
|
||||
temporary = MoEEpMegaLayer(
|
||||
bootstrap=self.bootstrap,
|
||||
fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity),
|
||||
weights=None,
|
||||
backend=backend,
|
||||
)
|
||||
payload = None
|
||||
try:
|
||||
temporary.warmup(tensors)
|
||||
if self.bootstrap.rank == 0:
|
||||
try:
|
||||
payload = json.loads(native_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
finally:
|
||||
temporary.destroy()
|
||||
payload = self._broadcast(payload)
|
||||
if not _valid_native_cache(payload):
|
||||
raise RuntimeError("MegaMoE autotune did not record its selected tactic")
|
||||
return payload
|
||||
|
||||
def _prepare_profile(self, mega, tensors, capacity: int) -> None:
|
||||
context = _active_context
|
||||
assert context is not None
|
||||
profile_key = f"{self._geometry_key}:{capacity}"
|
||||
profile_inputs = _profile_inputs(
|
||||
tensors, capacity, self.fleet_params.num_experts
|
||||
)
|
||||
native_path = context.staged_profiles.get(profile_key)
|
||||
needs_staging = native_path is None
|
||||
if needs_staging:
|
||||
native_path = (
|
||||
context.directory / f"profile{len(context.staged_profiles)}.json"
|
||||
)
|
||||
with _native_cache(native_path):
|
||||
if needs_staging:
|
||||
key = (
|
||||
self.bootstrap.process_group,
|
||||
torch.cuda.current_device(),
|
||||
profile_key,
|
||||
)
|
||||
payload = _session_winners.get(key)
|
||||
if payload is None:
|
||||
payload = self._broadcast(
|
||||
context.records.get(profile_key)
|
||||
if self.bootstrap.rank == 0
|
||||
else None
|
||||
)
|
||||
source = "cache hit"
|
||||
if not _valid_native_cache(payload):
|
||||
payload = self._tune(profile_inputs, capacity, native_path)
|
||||
source = "tuned"
|
||||
_session_winners[key] = payload
|
||||
if self.bootstrap.rank == 0:
|
||||
logger.info(
|
||||
"MegaMoE %s: capacity=%s, knobs=%s",
|
||||
source,
|
||||
capacity,
|
||||
payload["entries"][0]["knobs"],
|
||||
)
|
||||
# The backend owns dtype/layout keys and loads its original payload.
|
||||
try:
|
||||
native_path.write_text(json.dumps(payload))
|
||||
written = True
|
||||
except OSError:
|
||||
written = False
|
||||
self._require_all_ranks(
|
||||
written,
|
||||
"Cannot stage MegaMoE's selected tactic for workspace creation",
|
||||
)
|
||||
context.staged_profiles[profile_key] = native_path
|
||||
if context.records.get(profile_key) != payload:
|
||||
context.records[profile_key] = payload
|
||||
context.dirty = True
|
||||
workspace = mega.create_workspace(capacity)
|
||||
mega.warmup(profile_inputs, workspace=workspace)
|
||||
self.workspaces[capacity] = workspace
|
||||
|
||||
@staticmethod
|
||||
def _forward(mega, tensors, workspace=None):
|
||||
if mega.supports_output_view:
|
||||
return mega.forward(
|
||||
tensors, workspace=workspace, return_workspace_view=True
|
||||
)
|
||||
return mega.forward(tensors, workspace=workspace)
|
||||
|
||||
def __call__(self, mega, tensors):
|
||||
context = _active_context
|
||||
if context is None and not self.workspaces:
|
||||
self._fallback_used = True
|
||||
return self._forward(mega, tensors)
|
||||
if context is not None and not self.workspaces:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"MegaMoE autotune must complete before CUDA graph capture"
|
||||
)
|
||||
if self._fallback_used:
|
||||
raise RuntimeError(
|
||||
"MegaMoE autotune must precede its first ordinary forward"
|
||||
)
|
||||
import torch.distributed as dist
|
||||
|
||||
if (
|
||||
dist.is_initialized()
|
||||
and self.bootstrap.world_size != dist.get_world_size()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"FlashInfer MegaMoE startup tuning currently requires the full WORLD process group"
|
||||
)
|
||||
# The startup dummy can be much larger than captured decode batches.
|
||||
# Keep small-N profiles bounded by decode, plus one prefill profile.
|
||||
capacities = set(_capacities(context.decode_num_tokens))
|
||||
capacities.add(context.prefill_num_tokens)
|
||||
for capacity in sorted(capacities):
|
||||
self._prepare_profile(mega, tensors, capacity)
|
||||
self._profile_capacities = tuple(sorted(self.workspaces))
|
||||
|
||||
count = self._common_tokens(tensors)
|
||||
index = bisect_left(self._profile_capacities, count)
|
||||
if index < len(self._profile_capacities):
|
||||
return self._forward(
|
||||
mega, tensors, self.workspaces[self._profile_capacities[index]]
|
||||
)
|
||||
raise ValueError(
|
||||
f"MegaMoE received {count} tokens per rank, exceeding its prepared "
|
||||
f"decode/prefill capacity {max(self.workspaces)}"
|
||||
)
|
||||
@@ -19,6 +19,7 @@ import functools
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
@@ -34,8 +35,10 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
get_spec,
|
||||
max_prefill_buffer_tokens,
|
||||
max_speculative_num_draft_tokens,
|
||||
)
|
||||
from sglang.srt.utils import empty_context, log_info_on_rank0
|
||||
from sglang.srt.utils.common import get_eager_max_batch_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
@@ -94,6 +97,7 @@ def should_run_flashinfer_autotune(
|
||||
"flashinfer_mxfp4",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cutlass",
|
||||
"flashinfer_megamoe",
|
||||
]
|
||||
|
||||
from sglang.srt.layers.quantization.fp4_utils import (
|
||||
@@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path:
|
||||
/ cache_key
|
||||
)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
return (
|
||||
cache_dir
|
||||
/ f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json"
|
||||
)
|
||||
rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}"
|
||||
if (
|
||||
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
|
||||
and not mr.is_draft_worker
|
||||
):
|
||||
from sglang.srt.platforms import current_platform
|
||||
|
||||
# Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's
|
||||
# namespaced records currently require one writer per file. Follow the
|
||||
# P2P cache's device isolation, using physical UUIDs for stable identity
|
||||
# across CUDA index remapping and restarts. Engines sharing physical GPUs
|
||||
# must use separate SGLANG_CACHE_DIRs.
|
||||
rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}"
|
||||
return cache_dir / f"{rank_key}.json"
|
||||
|
||||
|
||||
def _autotune_tactic_sync_group(
|
||||
@@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
runs_dir = cache_path.parent / "runs"
|
||||
runs_dir.mkdir(parents=True, exist_ok=True)
|
||||
autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}"
|
||||
autotune_cache = runs_dir / (
|
||||
f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}"
|
||||
)
|
||||
logger.info(
|
||||
"Running FlashInfer autotune (cache reuse DISABLED via "
|
||||
"SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s",
|
||||
@@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
mr.forward_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.get_device_module(mr.device).stream(mr.forward_stream):
|
||||
from sglang.srt.layers.logits_processor import autotune_dummy_run_mode
|
||||
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
|
||||
megamoe_autotune_context,
|
||||
)
|
||||
|
||||
skip_ops = get_flashinfer_autotune_skip_ops(mr)
|
||||
# autotune(cache=...) clears all file-loaded tactics on entry, which would drop
|
||||
@@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
tuner = AutoTuner.get()
|
||||
if reuse_cache and autotune_cache.is_file():
|
||||
tuner.load_configs(str(autotune_cache))
|
||||
mega_context = empty_context()
|
||||
if (
|
||||
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
|
||||
and not mr.is_draft_worker
|
||||
and "flashinfer_megamoe" not in skip_ops
|
||||
):
|
||||
# The chunk size is already per DP rank. Prepare one prefill profile
|
||||
# independently of the optional full-model EXTEND autotune pass.
|
||||
prefill_num_tokens = (
|
||||
max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
|
||||
)
|
||||
mega_context = megamoe_autotune_context(
|
||||
decode_num_tokens=max(
|
||||
mr.max_decode_logits_rows(),
|
||||
get_eager_max_batch_size(mr.max_running_requests)
|
||||
* mr.decode_num_tokens_per_req(
|
||||
num_draft_tokens=max_speculative_num_draft_tokens()
|
||||
),
|
||||
),
|
||||
prefill_num_tokens=prefill_num_tokens,
|
||||
)
|
||||
with (
|
||||
_autotune_process_group(sync_group),
|
||||
autotune(
|
||||
@@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
cache=None if reuse_cache else str(autotune_cache),
|
||||
skip_ops=skip_ops,
|
||||
),
|
||||
mega_context,
|
||||
autotune_dummy_run_mode(run_lm_head=run_lm_head),
|
||||
):
|
||||
yield
|
||||
|
||||
@@ -644,7 +644,6 @@ class Resources(_FlagGroupBase):
|
||||
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
|
||||
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
|
||||
tbo_event_pool: dict = msgspec.field(default_factory=dict)
|
||||
flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
|
||||
# State capturers (installed by their subsystems when capture is on).
|
||||
indexer_capturer: Any = None
|
||||
experts_capturer: Any = None
|
||||
|
||||
@@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
|
||||
# max-based graph bucket). See #30432 re: the misleading flag name.
|
||||
return True
|
||||
elif moe_a2a_backend.is_flashinfer_megamoe():
|
||||
# MegaMoE's capacity profiles require the same decode graph
|
||||
# bucket on every EP rank. As for FlashInfer A2A above, this only
|
||||
# enables DP-count bookkeeping; expert inputs remain local.
|
||||
return True
|
||||
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
|
||||
# Same bookkeeping, for the same reason. Bounding mori's receive
|
||||
# buffer means baking a fan-in size into a captured graph, and the
|
||||
|
||||
Reference in New Issue
Block a user