[FlashInfer] Add startup autotuning for MegaMoE (#43789)
This commit is contained in:
@@ -164,7 +164,7 @@ SGLang supports various environment variables that can be used to configure its
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`SGLANG_FLASHINFER_AUTOTUNE_CACHE`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs. Set to `0` to force re-autotuning on every startup; the fresh result is written to a `runs/<rank>.<timestamp>.json` sibling file (the canonical cache is left untouched).</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Reuse persisted FlashInfer autotune results from `SGLANG_CACHE_DIR` across runs, including MegaMoE capacity profiles. Set to `0` to disable disk-cache reuse; the fresh result is written to a `runs/<rank>.<timestamp>.<uuid>.json` sibling file (the canonical cache is left untouched). Target MegaMoE rank files include the physical GPU UUID. Use separate `SGLANG_CACHE_DIR` roots for engines sharing physical GPUs; a running process retains tactics for already-pooled workspaces.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`true`</td>
|
||||
</tr>
|
||||
<tr>
|
||||
@@ -363,7 +363,7 @@ SGLang supports various environment variables that can be used to configure its
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE</code></td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Use the CuTe DSL MegaMOE in-kernel FC2 reduction. This can reduce workspace size and improve large-batch performance, but BF16 atomic accumulation is nondeterministic. It is incompatible with quantized combine dtypes.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Permit CuTe DSL MegaMOE in-kernel FC2 reduction for NVFP4 and MXFP8. FlashInfer's tuning profile may still select external reduction. BF16 atomic accumulation is nondeterministic; leave disabled for reproducibility. Requires BF16 combine.</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>"false"</code></td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
||||
@@ -1097,9 +1097,10 @@ class Envs:
|
||||
# (derived from cuda_graph_max_bs / chunked_prefill_size when unset).
|
||||
SGLANG_FLASHINFER_MEGAMOE_MAX_TOKENS_PER_RANK = EnvInt(0)
|
||||
# Opt-in in-kernel FC2 top-k reduce (cross-rank REDG atomic-add) for the
|
||||
# cutedsl mega kernels (NVFP4 / MXFP8). Deletes the multi-GB combine staging
|
||||
# region and can win at large batch, but makes the output accumulation order
|
||||
# nondeterministic (bf16 unordered sum) -- keep off for bit-reproducibility.
|
||||
# cutedsl mega kernels (NVFP4 / MXFP8). Permits in-kernel reduction;
|
||||
# FlashInfer's tuning profile may still select external reduction.
|
||||
# BF16 atomic accumulation is nondeterministic -- keep off for
|
||||
# bit-reproducibility.
|
||||
# No effect on the DeepGEMM (block-FP8) mega path, which lacks the knob.
|
||||
SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE = EnvBool(False)
|
||||
# Cross-rank combine wire format for the FlashInfer NVFP4 cutedsl MegaMOE
|
||||
|
||||
@@ -138,9 +138,7 @@ def _forward_megamoe_legacy(mega: Any, tensors: Any) -> torch.Tensor:
|
||||
|
||||
|
||||
def _select_megamoe_forward(mega: Any) -> Callable[[Any, Any], torch.Tensor]:
|
||||
import inspect
|
||||
|
||||
if "return_workspace_view" in inspect.signature(mega.forward).parameters:
|
||||
if mega.supports_output_view:
|
||||
return _forward_megamoe_with_workspace_view
|
||||
return _forward_megamoe_legacy
|
||||
|
||||
@@ -287,26 +285,46 @@ def _ensure_flashinfer_megamoe_layer(
|
||||
layer.hidden_size,
|
||||
)
|
||||
|
||||
process_group = None
|
||||
if world_size > 1:
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
process_group = get_parallel().moe_ep_group.device_group
|
||||
bootstrap = BootstrapConfig(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
device=torch.cuda.current_device(),
|
||||
process_group=process_group,
|
||||
)
|
||||
fleet_params = FleetParams(
|
||||
num_experts=layer.num_experts,
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
token_hidden_size=layer.hidden_size,
|
||||
)
|
||||
backend = MegaConfig(
|
||||
megakernel=megakernel_config,
|
||||
preprocess_weights=False,
|
||||
transformed_weights=transformed_weights,
|
||||
)
|
||||
mega = MoEEpMegaLayer(
|
||||
bootstrap=BootstrapConfig(
|
||||
world_size=world_size, rank=rank, device=torch.cuda.current_device()
|
||||
),
|
||||
fleet_params=FleetParams(
|
||||
num_experts=layer.num_experts,
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
token_hidden_size=layer.hidden_size,
|
||||
),
|
||||
bootstrap=bootstrap,
|
||||
fleet_params=fleet_params,
|
||||
# weights already preprocessed in prepare_*; with transformed_weights set
|
||||
# the kernel never reads `weights` (see MoEEpMegaLayer), so pass None.
|
||||
weights=None,
|
||||
backend=MegaConfig(
|
||||
megakernel=megakernel_config,
|
||||
preprocess_weights=False,
|
||||
transformed_weights=transformed_weights,
|
||||
),
|
||||
backend=backend,
|
||||
)
|
||||
layer._flashinfer_megamoe_layer = mega
|
||||
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
|
||||
if hasattr(megakernel_config, "knobs"):
|
||||
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
|
||||
MegaMoeTunedForward,
|
||||
)
|
||||
|
||||
layer._flashinfer_megamoe_forward = MegaMoeTunedForward(
|
||||
bootstrap, fleet_params, backend
|
||||
)
|
||||
else:
|
||||
layer._flashinfer_megamoe_forward = _select_megamoe_forward(mega)
|
||||
return mega
|
||||
|
||||
|
||||
@@ -360,12 +378,9 @@ def ensure_nvfp4_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
top_k=layer.top_k,
|
||||
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
|
||||
apply_topk_in_fc1=True,
|
||||
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
combine_dtype=resolve_flashinfer_megamoe_combine_dtype(),
|
||||
input_norm_const=input_norm_const,
|
||||
fc1_alpha=layer.g1_alphas,
|
||||
fc2_alpha=layer.g2_alphas,
|
||||
fc1_norm_const=layer.w2_input_scale_quant,
|
||||
),
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
@@ -386,7 +401,7 @@ def ensure_mxfp8_moe_layer_for_flashinfer_megamoe(layer: FusedMoE) -> Any:
|
||||
top_k=layer.top_k,
|
||||
kind="mxfp8_e4m3",
|
||||
gate_up_clamp=layer.moe_runner_config.swiglu_limit,
|
||||
in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
enable_in_kernel_fc2_reduce=envs.SGLANG_FLASHINFER_MEGAMOE_IN_KERNEL_FC2_REDUCE.get(),
|
||||
),
|
||||
w13_scale=layer.w13_weight_scale_inv,
|
||||
w2_scale=layer.w2_weight_scale_inv,
|
||||
@@ -536,49 +551,6 @@ def prepare_mxfp8_moe_weights_for_flashinfer_megamoe(
|
||||
)
|
||||
|
||||
|
||||
def _ensure_shared_workspace(mega: Any) -> None:
|
||||
"""Share this layer's workspace across MegaMOE layers with identical
|
||||
fleet/kernel geometry.
|
||||
FlashInfer's own workspace pool keys by fc1_alpha/fc2_alpha/fc1_norm_const
|
||||
tensor identity, but sglang binds distinct tensor objects per layer, so
|
||||
its pool never hits across layers; key on geometry instead, since those
|
||||
values are re-staged into the workspace per forward rather than baked
|
||||
into the compiled kernel. Creation is collective, so it must only happen
|
||||
on a layer's first forward, under warmup's cross-rank lockstep.
|
||||
"""
|
||||
if mega._workspace is not None:
|
||||
return
|
||||
fp = mega._fleet_params
|
||||
kc = mega._megakernel_config
|
||||
mc = mega._mega_config
|
||||
from sglang.srt.runtime_context import get_resources
|
||||
|
||||
key = (
|
||||
getattr(kc, "kernel_name", kc.__class__.__name__),
|
||||
mega._bootstrap.world_size,
|
||||
fp.num_experts,
|
||||
fp.max_tokens_per_rank,
|
||||
fp.token_hidden_size,
|
||||
kc.top_k,
|
||||
kc.intermediate_size,
|
||||
getattr(kc, "gate_up_clamp", None),
|
||||
getattr(kc, "activation_clamp", None),
|
||||
getattr(kc, "apply_topk_in_fc1", None),
|
||||
getattr(kc, "kind", None),
|
||||
getattr(kc, "in_kernel_fc2_reduce", None),
|
||||
getattr(kc, "combine_dtype", None),
|
||||
getattr(kc, "token_back_by_dispatch", None),
|
||||
getattr(kc, "fast_math", None),
|
||||
mc.quantize_input,
|
||||
)
|
||||
workspaces = get_resources().flashinfer_megamoe_workspaces
|
||||
shared = workspaces.get(key)
|
||||
if shared is None:
|
||||
workspaces[key] = mega._ensure_workspace()
|
||||
else:
|
||||
mega._workspace = shared
|
||||
|
||||
|
||||
@register_fused_func("flashinfer_megamoe", "flashinfer_megamoe")
|
||||
def run_flashinfer_megamoe(
|
||||
dispatch_output: DispatchOutput,
|
||||
@@ -599,7 +571,6 @@ def run_flashinfer_megamoe(
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids
|
||||
mega = quant_info.mega
|
||||
_ensure_shared_workspace(mega)
|
||||
|
||||
t = MoEEpTensors(
|
||||
hidden_states=x.to(torch.bfloat16),
|
||||
@@ -607,6 +578,7 @@ def run_flashinfer_megamoe(
|
||||
# directly into its final int64 workspace buffer. Keep this path copy-free.
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights.to(torch.float32),
|
||||
# Stage per-layer scales on every call so FlashInfer can pool workspaces.
|
||||
fc1_alpha=quant_info.fc1_alpha,
|
||||
fc2_alpha=quant_info.fc2_alpha,
|
||||
fc1_norm_const=quant_info.fc1_norm_const,
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Startup tuning and immutable capacity profiles for FlashInfer MegaMoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from bisect import bisect_left
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _AutotuneContext:
|
||||
decode_num_tokens: int
|
||||
prefill_num_tokens: int
|
||||
directory: Path
|
||||
records: dict
|
||||
staged_profiles: dict[str, Path] = field(default_factory=dict)
|
||||
dirty: bool = False
|
||||
|
||||
|
||||
_active_context: _AutotuneContext | None = None
|
||||
# Native workspace pooling spans startup contexts. Keep their
|
||||
# selected tactics consistent too; a cache-path change must not retune storage
|
||||
# already captured by another layer. These payloads own no weights or tensors.
|
||||
_session_winners: dict[tuple[Any, int, str], dict] = {}
|
||||
_CACHE_NAMESPACE = "sglang_flashinfer_megamoe"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def megamoe_autotune_context(
|
||||
decode_num_tokens: int,
|
||||
prefill_num_tokens: int,
|
||||
):
|
||||
"""Prepare profiles inside the engine's pre-capture FlashInfer autotune pass."""
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
global _active_context
|
||||
tuner = AutoTuner.get()
|
||||
previous = _active_context
|
||||
with TemporaryDirectory(prefix="sglang-megamoe-") as directory:
|
||||
context = _AutotuneContext(
|
||||
decode_num_tokens,
|
||||
prefill_num_tokens,
|
||||
Path(directory),
|
||||
tuner.get_namespaced_records(_CACHE_NAMESPACE),
|
||||
)
|
||||
_active_context = context
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_active_context = previous
|
||||
if context.dirty:
|
||||
tuner.publish_namespaced_records(_CACHE_NAMESPACE, context.records)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _native_cache(path: Path):
|
||||
# Native AUTO exposes its winner through this file; generic autotune owns
|
||||
# persistence. Each geometry is staged once per startup context.
|
||||
name = "FLASHINFER_MOE_EP_KNOB_CACHE"
|
||||
previous = os.environ.get(name)
|
||||
os.environ[name] = str(path)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if previous is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = previous
|
||||
|
||||
|
||||
def _capacities(limit: int) -> list[int]:
|
||||
limit = max(1, limit)
|
||||
return sorted({1 << i for i in range(limit.bit_length())} | {limit})
|
||||
|
||||
|
||||
def _profile_inputs(tensors, capacity: int, num_experts: int):
|
||||
count = tensors.num_tokens
|
||||
if count:
|
||||
|
||||
def resize(tensor):
|
||||
if capacity <= count:
|
||||
return tensor[:capacity]
|
||||
return tensor.repeat(
|
||||
((capacity + count - 1) // count,) + (1,) * (tensor.ndim - 1)
|
||||
)[:capacity]
|
||||
|
||||
hidden = resize(tensors.hidden_states)
|
||||
ids = resize(tensors.topk_ids)
|
||||
scores = resize(tensors.topk_weights)
|
||||
scales = resize(tensors.scales) if tensors.scales is not None else None
|
||||
else:
|
||||
# Idle ranks still enter every collective tuning iteration.
|
||||
hidden = tensors.hidden_states.new_zeros(
|
||||
(capacity, tensors.hidden_states.shape[1])
|
||||
)
|
||||
top_k = tensors.topk_ids.shape[1]
|
||||
ids = (
|
||||
torch.arange(
|
||||
capacity * top_k, dtype=tensors.topk_ids.dtype, device=hidden.device
|
||||
).reshape(capacity, top_k)
|
||||
% num_experts
|
||||
)
|
||||
scores = tensors.topk_weights.new_full((capacity, top_k), 1.0 / top_k)
|
||||
scales = (
|
||||
tensors.scales.new_ones((capacity, *tensors.scales.shape[1:]))
|
||||
if tensors.scales is not None
|
||||
else None
|
||||
)
|
||||
return replace(
|
||||
tensors, hidden_states=hidden, topk_ids=ids, topk_weights=scores, scales=scales
|
||||
)
|
||||
|
||||
|
||||
def _valid_native_cache(payload) -> bool:
|
||||
if not isinstance(payload, dict) or payload.get("version") != 1:
|
||||
return False
|
||||
entries = payload.get("entries")
|
||||
return (
|
||||
isinstance(entries, list)
|
||||
and len(entries) == 1
|
||||
and isinstance(entries[0], dict)
|
||||
and isinstance(entries[0].get("knobs"), dict)
|
||||
and bool(entries[0]["knobs"])
|
||||
)
|
||||
|
||||
|
||||
class MegaMoeTunedForward:
|
||||
"""Own public workspace handles; FlashInfer pools their storage across layers.
|
||||
|
||||
The caller installs this for backends with prepared weights and default
|
||||
``knobs=None``. Preparation must precede ordinary forwarding and capture.
|
||||
"""
|
||||
|
||||
def __init__(self, bootstrap, fleet_params, backend):
|
||||
if backend.megakernel.knobs is not None:
|
||||
raise ValueError("MegaMoE startup tuning requires knobs=None")
|
||||
if backend.preprocess_weights or backend.transformed_weights is None:
|
||||
raise ValueError("MegaMoE startup tuning requires prepared weights")
|
||||
self.bootstrap = bootstrap
|
||||
self.fleet_params = fleet_params
|
||||
self.backend = backend
|
||||
self.workspaces: dict[int, Any] = {}
|
||||
self._fallback_used = False
|
||||
self._profile_capacities: tuple[int, ...] = ()
|
||||
|
||||
def _common_tokens(self, tensors) -> int:
|
||||
if self.bootstrap.world_size == 1:
|
||||
return tensors.num_tokens
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
|
||||
counts = get_dp_global_num_tokens()
|
||||
if counts is None:
|
||||
raise RuntimeError(
|
||||
"MegaMoE workspace selection requires synchronized DP token counts"
|
||||
)
|
||||
return max(counts, default=0)
|
||||
|
||||
@cached_property
|
||||
def _geometry_key(self) -> str:
|
||||
config = {}
|
||||
for item in fields(self.backend.megakernel):
|
||||
# Per-layer scale values do not change the pooled kernel geometry.
|
||||
if item.name in (
|
||||
"knobs",
|
||||
"input_norm_const",
|
||||
"fc1_norm_const",
|
||||
"fc1_alpha",
|
||||
"fc2_alpha",
|
||||
):
|
||||
continue
|
||||
value = getattr(self.backend.megakernel, item.name)
|
||||
if not isinstance(value, torch.Tensor):
|
||||
config[item.name] = value
|
||||
# The enclosing flashinfer.autotune context validates runtime/compiler
|
||||
# metadata. This key only adds the MegaMoE configuration and geometry.
|
||||
return json.dumps(
|
||||
{
|
||||
"torch": torch.__version__,
|
||||
"mega_use_ncu": os.environ.get("MEGA_USE_NCU", "0"),
|
||||
"config": config,
|
||||
"world_size": self.bootstrap.world_size,
|
||||
"hidden": self.fleet_params.token_hidden_size,
|
||||
"num_experts": self.fleet_params.num_experts,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
|
||||
def _require_all_ranks(self, success: bool, message: str) -> None:
|
||||
if self.bootstrap.world_size > 1:
|
||||
import torch.distributed as dist
|
||||
|
||||
ready = torch.tensor(int(success), dtype=torch.int32, device="cuda")
|
||||
dist.all_reduce(
|
||||
ready, op=dist.ReduceOp.MIN, group=self.bootstrap.process_group
|
||||
)
|
||||
success = bool(ready.item())
|
||||
if not success:
|
||||
raise RuntimeError(message)
|
||||
|
||||
def _broadcast(self, value):
|
||||
if self.bootstrap.world_size == 1:
|
||||
return value
|
||||
import torch.distributed as dist
|
||||
|
||||
group = self.bootstrap.process_group
|
||||
values = [value]
|
||||
dist.broadcast_object_list(values, group=group, group_src=0)
|
||||
return values[0]
|
||||
|
||||
def _tune(self, tensors, capacity: int, native_path: Path):
|
||||
from flashinfer.moe_ep import MoEEpMegaLayer
|
||||
|
||||
backend = replace(
|
||||
self.backend, megakernel=replace(self.backend.megakernel, knobs="auto")
|
||||
)
|
||||
temporary = MoEEpMegaLayer(
|
||||
bootstrap=self.bootstrap,
|
||||
fleet_params=replace(self.fleet_params, max_tokens_per_rank=capacity),
|
||||
weights=None,
|
||||
backend=backend,
|
||||
)
|
||||
payload = None
|
||||
try:
|
||||
temporary.warmup(tensors)
|
||||
if self.bootstrap.rank == 0:
|
||||
try:
|
||||
payload = json.loads(native_path.read_text())
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
finally:
|
||||
temporary.destroy()
|
||||
payload = self._broadcast(payload)
|
||||
if not _valid_native_cache(payload):
|
||||
raise RuntimeError("MegaMoE autotune did not record its selected tactic")
|
||||
return payload
|
||||
|
||||
def _prepare_profile(self, mega, tensors, capacity: int) -> None:
|
||||
context = _active_context
|
||||
assert context is not None
|
||||
profile_key = f"{self._geometry_key}:{capacity}"
|
||||
profile_inputs = _profile_inputs(
|
||||
tensors, capacity, self.fleet_params.num_experts
|
||||
)
|
||||
native_path = context.staged_profiles.get(profile_key)
|
||||
needs_staging = native_path is None
|
||||
if needs_staging:
|
||||
native_path = (
|
||||
context.directory / f"profile{len(context.staged_profiles)}.json"
|
||||
)
|
||||
with _native_cache(native_path):
|
||||
if needs_staging:
|
||||
key = (
|
||||
self.bootstrap.process_group,
|
||||
torch.cuda.current_device(),
|
||||
profile_key,
|
||||
)
|
||||
payload = _session_winners.get(key)
|
||||
if payload is None:
|
||||
payload = self._broadcast(
|
||||
context.records.get(profile_key)
|
||||
if self.bootstrap.rank == 0
|
||||
else None
|
||||
)
|
||||
source = "cache hit"
|
||||
if not _valid_native_cache(payload):
|
||||
payload = self._tune(profile_inputs, capacity, native_path)
|
||||
source = "tuned"
|
||||
_session_winners[key] = payload
|
||||
if self.bootstrap.rank == 0:
|
||||
logger.info(
|
||||
"MegaMoE %s: capacity=%s, knobs=%s",
|
||||
source,
|
||||
capacity,
|
||||
payload["entries"][0]["knobs"],
|
||||
)
|
||||
# The backend owns dtype/layout keys and loads its original payload.
|
||||
try:
|
||||
native_path.write_text(json.dumps(payload))
|
||||
written = True
|
||||
except OSError:
|
||||
written = False
|
||||
self._require_all_ranks(
|
||||
written,
|
||||
"Cannot stage MegaMoE's selected tactic for workspace creation",
|
||||
)
|
||||
context.staged_profiles[profile_key] = native_path
|
||||
if context.records.get(profile_key) != payload:
|
||||
context.records[profile_key] = payload
|
||||
context.dirty = True
|
||||
workspace = mega.create_workspace(capacity)
|
||||
mega.warmup(profile_inputs, workspace=workspace)
|
||||
self.workspaces[capacity] = workspace
|
||||
|
||||
@staticmethod
|
||||
def _forward(mega, tensors, workspace=None):
|
||||
if mega.supports_output_view:
|
||||
return mega.forward(
|
||||
tensors, workspace=workspace, return_workspace_view=True
|
||||
)
|
||||
return mega.forward(tensors, workspace=workspace)
|
||||
|
||||
def __call__(self, mega, tensors):
|
||||
context = _active_context
|
||||
if context is None and not self.workspaces:
|
||||
self._fallback_used = True
|
||||
return self._forward(mega, tensors)
|
||||
if context is not None and not self.workspaces:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError(
|
||||
"MegaMoE autotune must complete before CUDA graph capture"
|
||||
)
|
||||
if self._fallback_used:
|
||||
raise RuntimeError(
|
||||
"MegaMoE autotune must precede its first ordinary forward"
|
||||
)
|
||||
import torch.distributed as dist
|
||||
|
||||
if (
|
||||
dist.is_initialized()
|
||||
and self.bootstrap.world_size != dist.get_world_size()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"FlashInfer MegaMoE startup tuning currently requires the full WORLD process group"
|
||||
)
|
||||
# The startup dummy can be much larger than captured decode batches.
|
||||
# Keep small-N profiles bounded by decode, plus one prefill profile.
|
||||
capacities = set(_capacities(context.decode_num_tokens))
|
||||
capacities.add(context.prefill_num_tokens)
|
||||
for capacity in sorted(capacities):
|
||||
self._prepare_profile(mega, tensors, capacity)
|
||||
self._profile_capacities = tuple(sorted(self.workspaces))
|
||||
|
||||
count = self._common_tokens(tensors)
|
||||
index = bisect_left(self._profile_capacities, count)
|
||||
if index < len(self._profile_capacities):
|
||||
return self._forward(
|
||||
mega, tensors, self.workspaces[self._profile_capacities[index]]
|
||||
)
|
||||
raise ValueError(
|
||||
f"MegaMoE received {count} tokens per rank, exceeding its prepared "
|
||||
f"decode/prefill capacity {max(self.workspaces)}"
|
||||
)
|
||||
@@ -19,6 +19,7 @@ import functools
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
@@ -34,8 +35,10 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
get_spec,
|
||||
max_prefill_buffer_tokens,
|
||||
max_speculative_num_draft_tokens,
|
||||
)
|
||||
from sglang.srt.utils import empty_context, log_info_on_rank0
|
||||
from sglang.srt.utils.common import get_eager_max_batch_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
@@ -94,6 +97,7 @@ def should_run_flashinfer_autotune(
|
||||
"flashinfer_mxfp4",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cutlass",
|
||||
"flashinfer_megamoe",
|
||||
]
|
||||
|
||||
from sglang.srt.layers.quantization.fp4_utils import (
|
||||
@@ -169,10 +173,20 @@ def flashinfer_autotune_cache_path(model_runner: ModelRunner) -> Path:
|
||||
/ cache_key
|
||||
)
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
return (
|
||||
cache_dir
|
||||
/ f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}.json"
|
||||
)
|
||||
rank_key = f"rank_tp{get_parallel().tp_rank}_pp{get_parallel().pp_rank}_dp{get_parallel().dp_rank or 0}"
|
||||
if (
|
||||
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
|
||||
and not mr.is_draft_worker
|
||||
):
|
||||
from sglang.srt.platforms import current_platform
|
||||
|
||||
# Ordinary FlashInfer tactics use best-effort merging, but MegaMoE's
|
||||
# namespaced records currently require one writer per file. Follow the
|
||||
# P2P cache's device isolation, using physical UUIDs for stable identity
|
||||
# across CUDA index remapping and restarts. Engines sharing physical GPUs
|
||||
# must use separate SGLANG_CACHE_DIRs.
|
||||
rank_key += f"_gpu{current_platform.get_device_uuid(mr.gpu_id)}"
|
||||
return cache_dir / f"{rank_key}.json"
|
||||
|
||||
|
||||
def _autotune_tactic_sync_group(
|
||||
@@ -265,7 +279,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
runs_dir = cache_path.parent / "runs"
|
||||
runs_dir.mkdir(parents=True, exist_ok=True)
|
||||
autotune_cache = runs_dir / f"{cache_path.stem}.{timestamp}{cache_path.suffix}"
|
||||
autotune_cache = runs_dir / (
|
||||
f"{cache_path.stem}.{timestamp}.{uuid.uuid4().hex}{cache_path.suffix}"
|
||||
)
|
||||
logger.info(
|
||||
"Running FlashInfer autotune (cache reuse DISABLED via "
|
||||
"SGLANG_FLASHINFER_AUTOTUNE_CACHE=0); writing fresh result to: %s",
|
||||
@@ -277,6 +293,9 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
mr.forward_stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.get_device_module(mr.device).stream(mr.forward_stream):
|
||||
from sglang.srt.layers.logits_processor import autotune_dummy_run_mode
|
||||
from sglang.srt.layers.moe.flashinfer_megamoe_autotune import (
|
||||
megamoe_autotune_context,
|
||||
)
|
||||
|
||||
skip_ops = get_flashinfer_autotune_skip_ops(mr)
|
||||
# autotune(cache=...) clears all file-loaded tactics on entry, which would drop
|
||||
@@ -284,6 +303,27 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
tuner = AutoTuner.get()
|
||||
if reuse_cache and autotune_cache.is_file():
|
||||
tuner.load_configs(str(autotune_cache))
|
||||
mega_context = empty_context()
|
||||
if (
|
||||
get_exec().moe.moe_runner_backend == "flashinfer_megamoe"
|
||||
and not mr.is_draft_worker
|
||||
and "flashinfer_megamoe" not in skip_ops
|
||||
):
|
||||
# The chunk size is already per DP rank. Prepare one prefill profile
|
||||
# independently of the optional full-model EXTEND autotune pass.
|
||||
prefill_num_tokens = (
|
||||
max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
|
||||
)
|
||||
mega_context = megamoe_autotune_context(
|
||||
decode_num_tokens=max(
|
||||
mr.max_decode_logits_rows(),
|
||||
get_eager_max_batch_size(mr.max_running_requests)
|
||||
* mr.decode_num_tokens_per_req(
|
||||
num_draft_tokens=max_speculative_num_draft_tokens()
|
||||
),
|
||||
),
|
||||
prefill_num_tokens=prefill_num_tokens,
|
||||
)
|
||||
with (
|
||||
_autotune_process_group(sync_group),
|
||||
autotune(
|
||||
@@ -291,6 +331,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool)
|
||||
cache=None if reuse_cache else str(autotune_cache),
|
||||
skip_ops=skip_ops,
|
||||
),
|
||||
mega_context,
|
||||
autotune_dummy_run_mode(run_lm_head=run_lm_head),
|
||||
):
|
||||
yield
|
||||
|
||||
@@ -644,7 +644,6 @@ class Resources(_FlagGroupBase):
|
||||
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
|
||||
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
|
||||
tbo_event_pool: dict = msgspec.field(default_factory=dict)
|
||||
flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
|
||||
# State capturers (installed by their subsystems when capture is on).
|
||||
indexer_capturer: Any = None
|
||||
experts_capturer: Any = None
|
||||
|
||||
@@ -4156,6 +4156,11 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
|
||||
# max-based graph bucket). See #30432 re: the misleading flag name.
|
||||
return True
|
||||
elif moe_a2a_backend.is_flashinfer_megamoe():
|
||||
# MegaMoE's capacity profiles require the same decode graph
|
||||
# bucket on every EP rank. As for FlashInfer A2A above, this only
|
||||
# enables DP-count bookkeeping; expert inputs remain local.
|
||||
return True
|
||||
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
|
||||
# Same bookkeeping, for the same reason. Bounding mori's receive
|
||||
# buffer means baking a fan-in size into a captured graph, and the
|
||||
|
||||
@@ -3,6 +3,7 @@ import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -69,6 +70,7 @@ def _load_megamoe_module(monkeypatch):
|
||||
for name, module in fake_modules.items():
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
|
||||
_load_autotune_module(monkeypatch)
|
||||
module_path = (
|
||||
Path(__file__).resolve().parents[5]
|
||||
/ "python/sglang/srt/layers/moe/flashinfer_megamoe.py"
|
||||
@@ -82,6 +84,269 @@ def _load_megamoe_module(monkeypatch):
|
||||
return module
|
||||
|
||||
|
||||
def _load_autotune_module(monkeypatch):
|
||||
name = "sglang.srt.layers.moe.flashinfer_megamoe_autotune"
|
||||
path = (
|
||||
Path(__file__).resolve().parents[5]
|
||||
/ "python/sglang/srt/layers/moe/flashinfer_megamoe_autotune.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _profile_fixture(monkeypatch, world_size=1, num_tokens=3):
|
||||
from dataclasses import dataclass
|
||||
|
||||
module = _load_autotune_module(monkeypatch)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
|
||||
monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
|
||||
tuner = types.SimpleNamespace(
|
||||
get_namespaced_records=lambda _: {},
|
||||
publish_namespaced_records=lambda *_: None,
|
||||
)
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"flashinfer.autotuner",
|
||||
types.SimpleNamespace(AutoTuner=types.SimpleNamespace(get=lambda: tuner)),
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class Tensors:
|
||||
hidden_states: torch.Tensor
|
||||
topk_ids: torch.Tensor
|
||||
topk_weights: torch.Tensor
|
||||
fc1_alpha: torch.Tensor
|
||||
scales: torch.Tensor | None = None
|
||||
|
||||
@property
|
||||
def num_tokens(self):
|
||||
return self.hidden_states.shape[0]
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
knobs: str | None = None
|
||||
kernel_name: str = "test_backend"
|
||||
input_norm_const: float = 1.0
|
||||
|
||||
@dataclass
|
||||
class Fleet:
|
||||
max_tokens_per_rank: int = 4
|
||||
num_experts: int = 4
|
||||
token_hidden_size: int = 4
|
||||
|
||||
@dataclass
|
||||
class Backend:
|
||||
megakernel: Config
|
||||
transformed_weights: object
|
||||
preprocess_weights: bool = False
|
||||
|
||||
tensors = Tensors(
|
||||
torch.ones(num_tokens, 4, dtype=torch.bfloat16),
|
||||
torch.arange(num_tokens * 2, dtype=torch.int32).reshape(num_tokens, 2) % 4,
|
||||
torch.full((num_tokens, 2), 0.5, dtype=torch.float32),
|
||||
torch.ones(4, dtype=torch.float32),
|
||||
)
|
||||
forward = module.MegaMoeTunedForward(
|
||||
types.SimpleNamespace(world_size=world_size, rank=0, process_group=None),
|
||||
Fleet(),
|
||||
Backend(Config(), object()),
|
||||
)
|
||||
calls = []
|
||||
|
||||
class Mega:
|
||||
supports_output_view = False
|
||||
|
||||
def forward(self, inputs, **kwargs):
|
||||
calls.append((inputs, kwargs.get("workspace")))
|
||||
return inputs.hidden_states
|
||||
|
||||
return module, forward, tensors, Mega(), calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"decode_tokens,expected", [(3, [1, 2, 3, 128]), (8, [1, 2, 4, 8, 128])]
|
||||
)
|
||||
def test_megamoe_startup_profiles_are_prepared_once(
|
||||
monkeypatch, decode_tokens, expected
|
||||
):
|
||||
module, forward, tensors, mega, calls = _profile_fixture(monkeypatch, num_tokens=96)
|
||||
prepared = []
|
||||
|
||||
def prepare(layer, inputs, capacity):
|
||||
prepared.append(capacity)
|
||||
forward.workspaces[capacity] = object()
|
||||
|
||||
monkeypatch.setattr(forward, "_prepare_profile", prepare)
|
||||
with module.megamoe_autotune_context(decode_tokens, prefill_num_tokens=128):
|
||||
assert forward(mega, tensors) is tensors.hidden_states
|
||||
# Bounds may exceed the bootstrap fleet. A large initial dummy must not
|
||||
# create intermediate prefill profiles.
|
||||
assert prepared == expected
|
||||
assert calls[-1][1] is forward.workspaces[128]
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
|
||||
for count, selected in ((2, 2), (decode_tokens, decode_tokens), (32, 128)):
|
||||
inputs = module._profile_inputs(tensors, count, num_experts=4)
|
||||
forward(mega, inputs)
|
||||
assert prepared == expected
|
||||
assert calls[-1][1] is forward.workspaces[selected]
|
||||
with pytest.raises(ValueError, match="exceeding its prepared"):
|
||||
forward(mega, module._profile_inputs(tensors, 129, num_experts=4))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("local_tokens", [0, 3])
|
||||
def test_megamoe_profile_selection_uses_all_dp_ranks(monkeypatch, local_tokens):
|
||||
module, forward, tensors, mega, calls = _profile_fixture(
|
||||
monkeypatch, world_size=4, num_tokens=local_tokens
|
||||
)
|
||||
dp = types.ModuleType("sglang.srt.layers.dp_attention")
|
||||
dp.get_dp_global_num_tokens = lambda: [0, 1, 3, 7]
|
||||
monkeypatch.setitem(sys.modules, dp.__name__, dp)
|
||||
forward.workspaces = {1: object(), 4: object(), 8: object(), 128: object()}
|
||||
forward._profile_capacities = tuple(forward.workspaces)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
|
||||
forward(mega, tensors)
|
||||
assert calls[-1][1] is forward.workspaces[8]
|
||||
|
||||
|
||||
def test_megamoe_without_tuning_preserves_capture_forward(monkeypatch):
|
||||
module, forward, tensors, mega, calls = _profile_fixture(monkeypatch)
|
||||
forward(mega, tensors)
|
||||
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: True)
|
||||
forward(mega, tensors)
|
||||
assert [workspace for _, workspace in calls] == [None, None]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("new_process", [False, True])
|
||||
def test_megamoe_profiles_reuse_cached_winner(monkeypatch, new_process):
|
||||
import json
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from unittest.mock import Mock
|
||||
|
||||
module, first, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=1)
|
||||
payload = {"version": 1, "entries": [{"knobs": {"native_tactic": 7}}]}
|
||||
events = []
|
||||
staged = []
|
||||
native_paths = []
|
||||
require_all_ranks = module.MegaMoeTunedForward._require_all_ranks
|
||||
readiness = Mock()
|
||||
|
||||
def require(self, success, message):
|
||||
readiness(success)
|
||||
require_all_ranks(self, success, message)
|
||||
|
||||
monkeypatch.setattr(module.MegaMoeTunedForward, "_require_all_ranks", require)
|
||||
|
||||
class Mega:
|
||||
supports_output_view = True
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.auto = kwargs.get("backend", first.backend).megakernel.knobs == "auto"
|
||||
|
||||
def warmup(self, inputs, workspace=None):
|
||||
if self.auto:
|
||||
events.append("tune")
|
||||
Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"]).write_text(
|
||||
json.dumps(payload)
|
||||
)
|
||||
else:
|
||||
assert workspace is self.workspace
|
||||
events.append("warmup")
|
||||
|
||||
def destroy(self):
|
||||
events.append("destroy")
|
||||
|
||||
def create_workspace(self, capacity):
|
||||
assert capacity == 1
|
||||
path = Path(os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"])
|
||||
native_paths.append(path)
|
||||
staged.append(json.loads(path.read_text()))
|
||||
self.workspace = object()
|
||||
events.append("create")
|
||||
return self.workspace
|
||||
|
||||
def forward(self, inputs, *, workspace, return_workspace_view=False):
|
||||
assert workspace is self.workspace
|
||||
assert return_workspace_view
|
||||
return inputs.hidden_states
|
||||
|
||||
fake_moe = types.ModuleType("flashinfer.moe_ep")
|
||||
fake_moe.MoEEpMegaLayer = Mega
|
||||
fake_autotuner = types.ModuleType("flashinfer.autotuner")
|
||||
records = {}
|
||||
tuner = types.SimpleNamespace(
|
||||
get_namespaced_records=Mock(
|
||||
side_effect=lambda namespace: records.get(namespace, {}).copy()
|
||||
),
|
||||
publish_namespaced_records=Mock(
|
||||
side_effect=lambda namespace, values: records.__setitem__(
|
||||
namespace, values.copy()
|
||||
)
|
||||
),
|
||||
)
|
||||
fake_autotuner.AutoTuner = types.SimpleNamespace(get=lambda: tuner)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer", types.ModuleType("flashinfer"))
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.moe_ep", fake_moe)
|
||||
monkeypatch.setitem(sys.modules, "flashinfer.autotuner", fake_autotuner)
|
||||
monkeypatch.setenv("FLASHINFER_MOE_EP_KNOB_CACHE", "original-cache.json")
|
||||
|
||||
owners = []
|
||||
for phase in range(2):
|
||||
if owners:
|
||||
if new_process:
|
||||
module._session_winners.clear() # Only the loaded cache survives.
|
||||
else:
|
||||
records.clear() # A new cache context must preserve pooled choices.
|
||||
with module.megamoe_autotune_context(
|
||||
decode_num_tokens=1,
|
||||
prefill_num_tokens=1,
|
||||
):
|
||||
for scale in (1.0, 2.0):
|
||||
backend = replace(
|
||||
first.backend,
|
||||
megakernel=replace(
|
||||
first.backend.megakernel, input_norm_const=scale
|
||||
),
|
||||
)
|
||||
forward = module.MegaMoeTunedForward(
|
||||
first.bootstrap, first.fleet_params, backend
|
||||
)
|
||||
mega = Mega()
|
||||
assert forward(mega, tensors) is tensors.hidden_states
|
||||
owners.append(mega)
|
||||
assert readiness.call_count == phase + 1
|
||||
assert native_paths[-2] == native_paths[-1]
|
||||
assert native_paths[-1].is_file()
|
||||
assert not native_paths[-1].exists()
|
||||
assert os.environ["FLASHINFER_MOE_EP_KNOB_CACHE"] == "original-cache.json"
|
||||
assert list(records["sglang_flashinfer_megamoe"].values()) == [payload]
|
||||
assert tuner.get_namespaced_records.call_count == phase + 1
|
||||
|
||||
assert tuner.publish_namespaced_records.call_count == (1 if new_process else 2)
|
||||
assert events == ["tune", "destroy"] + ["create", "warmup"] * 4
|
||||
assert staged == [payload] * 4
|
||||
assert len({id(owner.workspace) for owner in owners}) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens,capacity", [(0, 8), (3, 8), (3, 1)])
|
||||
def test_megamoe_profile_inputs_retain_runtime_scales(monkeypatch, tokens, capacity):
|
||||
module, _, tensors, _, _ = _profile_fixture(monkeypatch, num_tokens=tokens)
|
||||
tensors.scales = torch.ones(tokens, 2)
|
||||
profile = module._profile_inputs(tensors, capacity, num_experts=4)
|
||||
assert profile.hidden_states.shape == (capacity, 4)
|
||||
assert profile.topk_ids.shape == profile.topk_weights.shape == (capacity, 2)
|
||||
assert profile.hidden_states.dtype == torch.bfloat16
|
||||
assert profile.topk_ids.dtype == torch.int32
|
||||
assert profile.topk_weights.dtype == torch.float32
|
||||
assert profile.fc1_alpha is tensors.fc1_alpha
|
||||
assert profile.scales.shape == (capacity, 2)
|
||||
assert torch.all(profile.scales == 1)
|
||||
assert torch.all((profile.topk_ids >= 0) & (profile.topk_ids < 4))
|
||||
|
||||
|
||||
def test_max_tokens_uses_runtime_context_accessor(monkeypatch):
|
||||
module = _load_megamoe_module(monkeypatch)
|
||||
|
||||
@@ -112,7 +377,7 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
|
||||
output = torch.randn_like(hidden_states)
|
||||
|
||||
class Mega:
|
||||
_workspace = object()
|
||||
supports_output_view = False
|
||||
|
||||
def forward(self, tensors):
|
||||
self.tensors = tensors
|
||||
@@ -140,7 +405,8 @@ def test_adapter_keeps_router_ids_int32(monkeypatch):
|
||||
assert result.hidden_states is output
|
||||
|
||||
|
||||
def test_adapter_requests_workspace_output_view(monkeypatch):
|
||||
@pytest.mark.parametrize("supports_output_view", [False, True])
|
||||
def test_adapter_requests_workspace_output_view(monkeypatch, supports_output_view):
|
||||
module = _load_megamoe_module(monkeypatch)
|
||||
|
||||
class FakeMoEEpTensors:
|
||||
@@ -160,15 +426,13 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
|
||||
output = torch.randn_like(hidden_states)
|
||||
|
||||
class Mega:
|
||||
supports_output_view = True
|
||||
_workspace = object()
|
||||
|
||||
def forward(self, tensors, *, return_workspace_view=False):
|
||||
self.tensors = tensors
|
||||
self.return_workspace_view = return_workspace_view
|
||||
return output
|
||||
|
||||
mega = Mega()
|
||||
mega.supports_output_view = supports_output_view
|
||||
dispatch_output = types.SimpleNamespace(
|
||||
hidden_states=hidden_states,
|
||||
topk_output=types.SimpleNamespace(
|
||||
@@ -186,7 +450,7 @@ def test_adapter_requests_workspace_output_view(monkeypatch):
|
||||
assert result.hidden_states is output
|
||||
assert mega.tensors.topk_ids.data_ptr() == topk_ids.data_ptr()
|
||||
assert mega.tensors.topk_ids.dtype == torch.int32
|
||||
assert mega.return_workspace_view is True
|
||||
assert mega.return_workspace_view is supports_output_view
|
||||
|
||||
|
||||
def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
|
||||
@@ -215,6 +479,4 @@ def test_capture_safe_ue8m0_pack_is_scoped(monkeypatch):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
|
||||
@@ -14,6 +14,7 @@ register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-l
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
import unittest
|
||||
@@ -65,6 +66,146 @@ class TestAutotuneTacticSyncGroup(CustomTestCase):
|
||||
self.assertIsNone(_autotune_tactic_sync_group(tp_group))
|
||||
|
||||
|
||||
class TestMegaMoEAutotuneStartup(CustomTestCase):
|
||||
def test_context_bridges_profile_limits(self):
|
||||
from sglang.srt.layers.moe import flashinfer_megamoe_autotune as mega_autotune
|
||||
|
||||
runner = SimpleNamespace(
|
||||
device="cuda",
|
||||
forward_stream=Mock(),
|
||||
max_decode_logits_rows=Mock(),
|
||||
decode_num_tokens_per_req=Mock(),
|
||||
max_running_requests=256,
|
||||
is_draft_worker=False,
|
||||
)
|
||||
general = Mock(side_effect=lambda *_args, **_kwargs: nullcontext())
|
||||
modules = {
|
||||
"flashinfer.autotuner": SimpleNamespace(
|
||||
AutoTuner=SimpleNamespace(
|
||||
get=lambda: SimpleNamespace(
|
||||
get_namespaced_records=lambda _: {},
|
||||
load_configs=lambda _: None,
|
||||
save_configs=lambda _: None,
|
||||
publish_namespaced_records=lambda *_: None,
|
||||
)
|
||||
),
|
||||
_collect_metadata=lambda: ENV,
|
||||
autotune=general,
|
||||
get_autotune_process_group=lambda: None,
|
||||
set_autotune_process_group=lambda _: None,
|
||||
),
|
||||
"sglang.srt.layers.logits_processor": SimpleNamespace(
|
||||
autotune_dummy_run_mode=lambda **_: nullcontext()
|
||||
),
|
||||
}
|
||||
# The resolved chunk size is already per DP rank. Disabled chunking
|
||||
# uses the scheduler's prefill ceiling; skip-op suppresses the bridge.
|
||||
# (chunk, skip, draft, graph rows, maximum MTP width, decode, prefill)
|
||||
cases = [
|
||||
(4096, False, False, 128 * 6, 6, 256 * 6, 4096),
|
||||
(-1, False, False, 8192, 6, 8192, 16384),
|
||||
(4096, False, False, 4096, 32, 8192, 4096),
|
||||
(4096, True, False, 128 * 6, 6, None, None),
|
||||
(4096, False, True, 128 * 6, 6, None, None),
|
||||
]
|
||||
previous = mega_autotune._active_context
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
cache_path = Path(directory) / "rank.json"
|
||||
for chunk_size, skip, draft, graph_rows, width, decode, prefill in cases:
|
||||
runner.is_draft_worker = draft
|
||||
runner.max_decode_logits_rows.return_value = graph_rows
|
||||
runner.decode_num_tokens_per_req.return_value = width
|
||||
skip_ops = {"flashinfer_megamoe"} if skip else set()
|
||||
with (
|
||||
self.subTest(chunk_size=chunk_size, skip=skip, draft=draft),
|
||||
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
|
||||
patch.dict(sys.modules, modules),
|
||||
patch.dict(
|
||||
os.environ,
|
||||
SGLANG_FLASHINFER_AUTOTUNE_CACHE="1",
|
||||
),
|
||||
patch.multiple(
|
||||
autotune,
|
||||
flashinfer_autotune_cache_path=lambda _: cache_path,
|
||||
get_flashinfer_autotune_skip_ops=lambda _: skip_ops,
|
||||
get_exec=lambda: SimpleNamespace(
|
||||
moe=SimpleNamespace(moe_runner_backend="flashinfer_megamoe")
|
||||
),
|
||||
get_eager_max_batch_size=lambda bs: bs,
|
||||
max_speculative_num_draft_tokens=lambda: width,
|
||||
max_prefill_buffer_tokens=lambda: max(chunk_size, 0),
|
||||
get_schedule=lambda: SimpleNamespace(max_prefill_tokens=16384),
|
||||
),
|
||||
patch.object(
|
||||
autotune.torch.cuda,
|
||||
"current_stream",
|
||||
return_value=Mock(),
|
||||
),
|
||||
patch.object(
|
||||
autotune.torch,
|
||||
"get_device_module",
|
||||
return_value=SimpleNamespace(stream=lambda _: nullcontext()),
|
||||
),
|
||||
):
|
||||
general.reset_mock()
|
||||
with autotune.flashinfer_autotune_context(
|
||||
runner, run_lm_head=False
|
||||
):
|
||||
general.assert_called_once_with(
|
||||
True,
|
||||
cache=None,
|
||||
skip_ops=skip_ops,
|
||||
)
|
||||
context = mega_autotune._active_context
|
||||
if prefill is None:
|
||||
self.assertIs(context, previous)
|
||||
else:
|
||||
self.assertIsNot(context, previous)
|
||||
runner.max_decode_logits_rows.assert_called_with()
|
||||
runner.decode_num_tokens_per_req.assert_called_with(
|
||||
num_draft_tokens=width
|
||||
)
|
||||
self.assertEqual(context.decode_num_tokens, decode)
|
||||
self.assertEqual(context.prefill_num_tokens, prefill)
|
||||
self.assertIs(mega_autotune._active_context, previous)
|
||||
|
||||
# Cache-disabled startups in the same second must still write to
|
||||
# different files, even if they use the same device and rank.
|
||||
with (
|
||||
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
|
||||
patch.dict(sys.modules, modules),
|
||||
patch.dict(os.environ, SGLANG_FLASHINFER_AUTOTUNE_CACHE="0"),
|
||||
patch.multiple(
|
||||
autotune,
|
||||
flashinfer_autotune_cache_path=lambda _: cache_path,
|
||||
get_flashinfer_autotune_skip_ops=lambda _: set(),
|
||||
get_exec=lambda: SimpleNamespace(
|
||||
moe=SimpleNamespace(moe_runner_backend="triton")
|
||||
),
|
||||
),
|
||||
patch.object(autotune.datetime, "datetime") as clock,
|
||||
patch.object(
|
||||
autotune.torch.cuda, "current_stream", return_value=Mock()
|
||||
),
|
||||
patch.object(
|
||||
autotune.torch,
|
||||
"get_device_module",
|
||||
return_value=SimpleNamespace(stream=lambda _: nullcontext()),
|
||||
),
|
||||
):
|
||||
clock.now.return_value.strftime.return_value = "20260922_010000"
|
||||
paths = []
|
||||
for _ in range(2):
|
||||
with autotune.flashinfer_autotune_context(
|
||||
runner, run_lm_head=False
|
||||
):
|
||||
paths.append(Path(general.call_args.kwargs["cache"]))
|
||||
self.assertNotEqual(*paths)
|
||||
for path in paths:
|
||||
self.assertEqual(path.parent, cache_path.parent / "runs")
|
||||
self.assertTrue(path.name.startswith("rank.20260922_010000."))
|
||||
|
||||
|
||||
class TestAutotuneCacheDigest(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
@@ -249,6 +390,13 @@ class TestAutotuneCachePhases(CustomTestCase):
|
||||
patch.object(
|
||||
autotune, "get_flashinfer_autotune_skip_ops", return_value=set()
|
||||
),
|
||||
patch.object(
|
||||
autotune,
|
||||
"get_exec",
|
||||
return_value=SimpleNamespace(
|
||||
moe=SimpleNamespace(moe_runner_backend="flashinfer_cutedsl")
|
||||
),
|
||||
),
|
||||
autotune.envs.SGLANG_FLASHINFER_AUTOTUNE_CACHE.override(True),
|
||||
get_parallel().override(tp_group=SimpleNamespace(world_size=1)),
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user