[AMD] Integrate Aiter MegaMoEv2 for DeepSeek-V4 (#35619)
This commit is contained in:
@@ -36,11 +36,15 @@ def check_mega_moe_compat(server_args: ServerArgs) -> None:
|
||||
"--enable-single-batch-overlap."
|
||||
)
|
||||
platform = get_platform()
|
||||
if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)):
|
||||
if not (
|
||||
(platform.is_cuda and (platform.is_sm90 or platform.is_sm100))
|
||||
or platform.is_hip
|
||||
):
|
||||
raise ValueError(
|
||||
"--moe-a2a-backend megamoe needs a CUDA SM90 GPU (block-FP8 experts) "
|
||||
"or an SM100-class GPU (MXFP4 / NVFP4 experts); it runs DeepGEMM "
|
||||
"kernels over CUDA symmetric memory."
|
||||
"kernels over CUDA symmetric memory. On ROCm, set "
|
||||
"SGLANG_AMD_USE_FLYDSL_MEGA_MOE=1 to select aiter MegaMoEv2 instead."
|
||||
)
|
||||
|
||||
|
||||
@@ -98,6 +102,8 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) ->
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.moe_a2a_backend != "megamoe":
|
||||
return
|
||||
if get_platform().is_hip:
|
||||
return
|
||||
|
||||
max_tokens_per_rank = (
|
||||
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
|
||||
|
||||
@@ -909,6 +909,12 @@ class Envs:
|
||||
SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256)
|
||||
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang")
|
||||
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
|
||||
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
|
||||
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
|
||||
SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("")
|
||||
SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False)
|
||||
SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False)
|
||||
SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD = EnvBool(False)
|
||||
# Above 8192 tokens of context, aiter's non-static workspace is large enough
|
||||
# that mem_fraction_static is scaled by 0.85 to leave room for it. Set this to
|
||||
# honor an explicitly passed --mem-fraction-static instead. Off by default:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
|
||||
try:
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
_HAS_TRITON = True
|
||||
except ImportError:
|
||||
_HAS_TRITON = False
|
||||
|
||||
|
||||
if _HAS_TRITON:
|
||||
|
||||
@triton.jit
|
||||
def _eplb_map_record_hist_kernel(
|
||||
topk_ids_ptr,
|
||||
dispatch_ptr,
|
||||
out_ids_ptr,
|
||||
load_ptr,
|
||||
num_valid_tokens_ptr,
|
||||
num_logical,
|
||||
num_physical,
|
||||
numel,
|
||||
HAS_DISPATCH: tl.constexpr,
|
||||
HAS_VALID_COUNT: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BINS: tl.constexpr,
|
||||
TOP_K: tl.constexpr,
|
||||
):
|
||||
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < numel
|
||||
logical_id = tl.load(topk_ids_ptr + offs, mask=mask, other=-1).to(tl.int64)
|
||||
valid_logical = (logical_id >= 0) & (logical_id < num_logical)
|
||||
safe_logical = tl.where(valid_logical, logical_id, 0)
|
||||
if HAS_DISPATCH:
|
||||
physical_id = tl.load(
|
||||
dispatch_ptr + safe_logical,
|
||||
mask=mask & valid_logical,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
else:
|
||||
physical_id = logical_id
|
||||
output_id = tl.where(valid_logical, physical_id, logical_id)
|
||||
tl.store(out_ids_ptr + offs, output_id, mask=mask)
|
||||
|
||||
valid_row = mask
|
||||
if HAS_VALID_COUNT:
|
||||
num_valid_tokens = tl.load(num_valid_tokens_ptr).to(tl.int64)
|
||||
valid_row &= (offs // TOP_K) < num_valid_tokens
|
||||
in_range = (
|
||||
valid_row
|
||||
& valid_logical
|
||||
& (physical_id >= 0)
|
||||
& (physical_id < num_physical)
|
||||
)
|
||||
bin_idx = tl.where(in_range, physical_id, num_physical).to(tl.int32)
|
||||
histogram = tl.histogram(bin_idx, NUM_BINS)
|
||||
bins = tl.arange(0, NUM_BINS)
|
||||
tl.atomic_add(load_ptr + bins, histogram, mask=bins < num_physical)
|
||||
|
||||
|
||||
def eplb_map_and_record_fused(
|
||||
topk_ids: torch.Tensor,
|
||||
dispatch_info: Optional[ExpertLocationDispatchInfo],
|
||||
load_buffer: torch.Tensor,
|
||||
num_token_non_padded: Optional[torch.Tensor],
|
||||
) -> Optional[torch.Tensor]:
|
||||
if not _HAS_TRITON or not topk_ids.is_cuda or topk_ids.numel() == 0:
|
||||
return None
|
||||
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm != "static":
|
||||
return None
|
||||
|
||||
dispatch = (
|
||||
dispatch_info.partial_logical_to_rank_dispatch_physical_map
|
||||
if dispatch_info is not None
|
||||
else None
|
||||
)
|
||||
if dispatch_info is not None and dispatch is None:
|
||||
return None
|
||||
num_physical = int(load_buffer.numel())
|
||||
num_logical = int(dispatch.numel()) if dispatch is not None else num_physical
|
||||
top_k = int(topk_ids.shape[-1])
|
||||
numel = topk_ids.numel()
|
||||
output = torch.empty_like(topk_ids)
|
||||
dispatch_ptr = dispatch if dispatch is not None else topk_ids
|
||||
valid_count_ptr = (
|
||||
num_token_non_padded if num_token_non_padded is not None else topk_ids
|
||||
)
|
||||
grid = (triton.cdiv(numel, 256),)
|
||||
_eplb_map_record_hist_kernel[grid](
|
||||
topk_ids.contiguous(),
|
||||
dispatch_ptr,
|
||||
output,
|
||||
load_buffer,
|
||||
valid_count_ptr,
|
||||
num_logical,
|
||||
num_physical,
|
||||
numel,
|
||||
HAS_DISPATCH=dispatch is not None,
|
||||
HAS_VALID_COUNT=num_token_non_padded is not None,
|
||||
BLOCK=256,
|
||||
NUM_BINS=1 << num_physical.bit_length(),
|
||||
TOP_K=top_k,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
__all__ = ["eplb_map_and_record_fused"]
|
||||
@@ -39,7 +39,7 @@ import torch
|
||||
import torch.distributed
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
STAT_LOGGER_ROLE_EXPERT_DISPATCH,
|
||||
ExpertDispatchCollector,
|
||||
@@ -48,6 +48,7 @@ from sglang.srt.observability.metrics_collector import (
|
||||
from sglang.srt.runtime_context import get_device as get_device_namespace
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_platform,
|
||||
get_schedule,
|
||||
logs_expert_balancedness_to_server_log,
|
||||
reports_expert_balancedness,
|
||||
@@ -59,6 +60,57 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# AMD specific function
|
||||
def megamoe_prefill_only_recorder_enabled() -> bool:
|
||||
# The prefill-only arm belongs to aiter MegaMoEv2, which only exists on ROCm.
|
||||
# Read the backend through the resolved accessor, not off a supplied
|
||||
# ServerArgs: test_supplied_instance_exposure_ratchet pins (file, field)
|
||||
# pairs where a `server_args` parameter is read for a field that resolution
|
||||
# fills in, and that surface may only shrink.
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
|
||||
return (
|
||||
get_platform().is_hip
|
||||
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
|
||||
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
|
||||
and get_moe_a2a_backend().is_megamoe()
|
||||
)
|
||||
|
||||
|
||||
# non-hip platform always return True
|
||||
def should_record_forward_pass(forward_batch: ForwardBatch) -> bool:
|
||||
if not megamoe_prefill_only_recorder_enabled():
|
||||
return True
|
||||
if _is_model_capture_mode():
|
||||
return False
|
||||
if forward_batch.forward_mode not in (
|
||||
ForwardMode.EXTEND,
|
||||
ForwardMode.SPLIT_PREFILL,
|
||||
):
|
||||
return False
|
||||
return int(forward_batch.extend_num_tokens or 0) > 0
|
||||
|
||||
|
||||
# non-hip platform always return True
|
||||
def should_advance_eplb_counter(forward_batch: ForwardBatch) -> bool:
|
||||
"""Whether a finished pass counts towards the EPLB rebalance interval.
|
||||
|
||||
Unlike `should_record_forward_pass`, which is local to this rank, the
|
||||
counter has to advance in lockstep across ranks, so it keys off the globally
|
||||
synchronized `is_extend_in_batch` rather than this rank's forward mode.
|
||||
"""
|
||||
if not megamoe_prefill_only_recorder_enabled():
|
||||
return True
|
||||
return forward_batch.is_extend_in_batch
|
||||
|
||||
|
||||
def _is_model_capture_mode() -> bool:
|
||||
from sglang.srt.model_executor.runner_utils.capture_mode import is_capture_mode
|
||||
|
||||
return is_capture_mode
|
||||
|
||||
|
||||
# --------------------------------------- Entrypoint -----------------------------------------
|
||||
|
||||
_OutputMode = Literal["file", "object"]
|
||||
@@ -117,6 +169,9 @@ class ExpertDistributionRecorder(ABC):
|
||||
def on_select_experts(self, topk_ids: torch.Tensor):
|
||||
pass
|
||||
|
||||
def get_current_pass_count_buffer(self, layer_idx: int):
|
||||
return None
|
||||
|
||||
def on_deepep_dispatch_normal(
|
||||
self,
|
||||
local_physical_count_of_layer: List[int],
|
||||
@@ -164,6 +219,7 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
|
||||
|
||||
self._recording = False
|
||||
self._disable_all = False
|
||||
self._record_current_pass = False
|
||||
self._current_forward_pass_id = Withable()
|
||||
self._current_layer_idx = Withable()
|
||||
self._current_debug_name = Withable()
|
||||
@@ -208,13 +264,17 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
|
||||
|
||||
def _on_forward_pass_start(self, forward_batch: ForwardBatch):
|
||||
if not self._recording:
|
||||
self._record_current_pass = False
|
||||
return
|
||||
self._record_current_pass = should_record_forward_pass(forward_batch)
|
||||
if not self._record_current_pass:
|
||||
return
|
||||
for gatherer_key, gatherer in self._single_pass_gatherers.items():
|
||||
gatherer.reset()
|
||||
gatherer.on_forward_pass_start(forward_batch)
|
||||
|
||||
def _on_forward_pass_end(self, forward_pass_id: int, outputs: Dict[str, Any]):
|
||||
if not self._recording:
|
||||
if not self._recording or not self._record_current_pass:
|
||||
return
|
||||
for gatherer_key, gatherer in self._single_pass_gatherers.items():
|
||||
single_pass_data = gatherer.collect()
|
||||
@@ -225,6 +285,14 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
|
||||
def on_select_experts(self, topk_ids: torch.Tensor):
|
||||
self._on_hook("on_select_experts", topk_ids=topk_ids)
|
||||
|
||||
def get_current_pass_count_buffer(self, layer_idx: int):
|
||||
if not self._recording or not self._record_current_pass:
|
||||
return None
|
||||
gatherer = self._single_pass_gatherers.get(_SINGLE_PASS_GATHERER_KEY_PRIMARY)
|
||||
if not isinstance(gatherer, _SelectExpertsSinglePassGatherer):
|
||||
return None
|
||||
return gatherer._data[layer_idx]
|
||||
|
||||
def on_deepep_dispatch_normal(
|
||||
self,
|
||||
local_physical_count_of_layer: List[int],
|
||||
@@ -251,6 +319,8 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
|
||||
def _on_hook(self, hook_name: str, **kwargs):
|
||||
if self._disable_all:
|
||||
return
|
||||
if megamoe_prefill_only_recorder_enabled() and not self._record_current_pass:
|
||||
return
|
||||
if not (
|
||||
self._recording or torch.get_device_module().is_current_stream_capturing()
|
||||
):
|
||||
@@ -548,7 +618,9 @@ class _SelectExpertsSinglePassGatherer(_LayerBasedGpuSinglePassGatherer):
|
||||
# can optimize (e.g. fuse / compile)
|
||||
def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor):
|
||||
topk_ids = topk_ids.flatten()
|
||||
mask = topk_ids != -1
|
||||
mask = (topk_ids >= 0) & (
|
||||
topk_ids < self._expert_location_metadata.num_physical_experts
|
||||
)
|
||||
self._data[layer_idx, :].scatter_add_(
|
||||
dim=0, index=topk_ids.masked_fill(~mask, 0).long(), src=mask.int()
|
||||
)
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.models.deepseek_common.utils import _device_sm
|
||||
from sglang.srt.runtime_context import get_exec
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deep_gemm import SymmBuffer
|
||||
@@ -43,6 +44,13 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
_MEGA_MOE_SYMM_BUFFER: dict = {}
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
def _use_amd_flydsl_mega_moe() -> bool:
|
||||
# aiter MegaMoEv2 exists only on ROCm; the platform check keeps the env from
|
||||
# diverting a CUDA run into a path whose kernels it does not have.
|
||||
return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
|
||||
|
||||
|
||||
def _mega_moe_mma_type(experts=None) -> str:
|
||||
@@ -152,6 +160,13 @@ def is_mega_moe_experts_ready(experts) -> bool:
|
||||
|
||||
|
||||
def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool:
|
||||
if _use_amd_flydsl_mega_moe():
|
||||
from sglang.srt.layers.moe.mega_moe_flydsl import (
|
||||
should_use_mega_moe as should_use_flydsl_mega_moe,
|
||||
)
|
||||
|
||||
return should_use_flydsl_mega_moe(moe, hidden_states)
|
||||
|
||||
if not get_moe_a2a_backend().is_megamoe():
|
||||
return False
|
||||
if not is_mega_moe_experts_ready(moe.experts):
|
||||
@@ -174,6 +189,15 @@ def forward_mega_moe(
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
input_ids_global: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
if _use_amd_flydsl_mega_moe():
|
||||
from sglang.srt.layers.moe.mega_moe_flydsl import (
|
||||
forward_mega_moe as forward_flydsl_mega_moe,
|
||||
)
|
||||
|
||||
return forward_flydsl_mega_moe(
|
||||
moe, hidden_states, forward_batch, input_ids_global
|
||||
)
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
|
||||
sbo_overlap_flag = (
|
||||
@@ -432,6 +456,14 @@ def _transpose_mega_moe_sf_for_utccp(sf: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
|
||||
def build_mega_moe_experts_weights(experts) -> None:
|
||||
if _use_amd_flydsl_mega_moe():
|
||||
from sglang.srt.layers.moe.mega_moe_flydsl import (
|
||||
build_mega_moe_experts_weights as build_flydsl_mega_moe_weights,
|
||||
)
|
||||
|
||||
build_flydsl_mega_moe_weights(experts)
|
||||
return
|
||||
|
||||
from deep_gemm import (
|
||||
transform_sf_into_required_layout,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
# Copyright 2023-2024 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0.
|
||||
"""Aiter MegaMoEV2 adapter for the SGLang MegaMoE hooks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE
|
||||
|
||||
|
||||
_MORI_SHMEM_READY = False
|
||||
_MEGA_MOE_INSTANCE: dict = {}
|
||||
|
||||
|
||||
def _import_aiter_megamoe():
|
||||
try:
|
||||
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
|
||||
from aiter.ops.shuffle import shuffle_scale_a16w4, shuffle_weight_a16w4
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"Aiter MegaMoE requires a package containing MegaMoEV2 and the "
|
||||
f"A16W4 shuffle operators. Original import error: {exc}"
|
||||
) from exc
|
||||
|
||||
return SimpleNamespace(
|
||||
op=MegaMoEV2,
|
||||
shuffle_weight=shuffle_weight_a16w4,
|
||||
shuffle_scale=shuffle_scale_a16w4,
|
||||
)
|
||||
|
||||
|
||||
def _mtpr() -> int:
|
||||
mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get())
|
||||
if mtpr <= 0 or mtpr & (mtpr - 1):
|
||||
raise ValueError(
|
||||
f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr} must be a positive power of two"
|
||||
)
|
||||
return mtpr
|
||||
|
||||
|
||||
def _ep_rank_world():
|
||||
parallel = get_parallel()
|
||||
return parallel.moe_ep_rank, parallel.moe_ep_size
|
||||
|
||||
|
||||
def _ensure_mori_shmem() -> None:
|
||||
global _MORI_SHMEM_READY
|
||||
if _MORI_SHMEM_READY:
|
||||
return
|
||||
|
||||
import mori.shmem
|
||||
|
||||
group_name = "megamoe_aiter"
|
||||
cpu_group = get_parallel().moe_ep_group.cpu_group
|
||||
try:
|
||||
torch._C._distributed_c10d._register_process_group(group_name, cpu_group)
|
||||
except Exception as exc:
|
||||
if "already registered" not in str(exc):
|
||||
raise
|
||||
mori.shmem.shmem_torch_process_group_init(group_name)
|
||||
_MORI_SHMEM_READY = True
|
||||
|
||||
|
||||
def build_mega_moe_experts_weights(layer) -> None:
|
||||
if getattr(layer, "_mega_moe_weights_built", False):
|
||||
return
|
||||
|
||||
backend = _import_aiter_megamoe()
|
||||
|
||||
def scale_param(name):
|
||||
scale = getattr(layer, name, None)
|
||||
if scale is None:
|
||||
scale = getattr(layer, name + "_inv", None)
|
||||
if scale is None:
|
||||
raise AttributeError(f"Aiter MegaMoE weight scale {name} is missing")
|
||||
return scale
|
||||
|
||||
def shuffle_existing(weight, scale, *, gate_up):
|
||||
experts = weight.shape[0]
|
||||
shuffled_weight = backend.shuffle_weight(weight, 16, gate_up).contiguous()
|
||||
shuffled_scale = backend.shuffle_scale(
|
||||
scale.view(-1, scale.shape[-1]), experts, gate_up
|
||||
).contiguous()
|
||||
return (
|
||||
shuffled_weight.view(torch.uint8).contiguous(),
|
||||
shuffled_scale.view(torch.uint8).contiguous(),
|
||||
)
|
||||
|
||||
w13_scale = scale_param("w13_weight_scale")
|
||||
w2_scale = scale_param("w2_weight_scale")
|
||||
layer._mega_w1, layer._mega_w1_scale = shuffle_existing(
|
||||
layer.w13_weight.data, w13_scale.data, gate_up=True
|
||||
)
|
||||
layer._mega_w2, layer._mega_w2_scale = shuffle_existing(
|
||||
layer.w2_weight.data, w2_scale.data, gate_up=False
|
||||
)
|
||||
|
||||
experts = layer.w13_weight.shape[0]
|
||||
layer.w13_weight.data = layer._mega_w1.view(experts, -1)
|
||||
w13_scale.data = layer._mega_w1_scale.view(experts, -1)
|
||||
layer.w2_weight.data = layer._mega_w2.view(experts, -1)
|
||||
w2_scale.data = layer._mega_w2_scale.view(experts, -1)
|
||||
layer._mega_moe_weights_built = True
|
||||
|
||||
|
||||
def _get_or_build_mega_moe(
|
||||
layer,
|
||||
*,
|
||||
model_dim: int,
|
||||
inter_dim: int,
|
||||
experts: int,
|
||||
topk: int,
|
||||
quant: str,
|
||||
swiglu_limit: float,
|
||||
mtpr: int,
|
||||
):
|
||||
_ensure_mori_shmem()
|
||||
backend = _import_aiter_megamoe()
|
||||
rank, world = _ep_rank_world()
|
||||
key = (
|
||||
rank,
|
||||
world,
|
||||
model_dim,
|
||||
inter_dim,
|
||||
experts,
|
||||
topk,
|
||||
quant,
|
||||
swiglu_limit,
|
||||
mtpr,
|
||||
)
|
||||
mega = _MEGA_MOE_INSTANCE.get(key)
|
||||
if mega is None:
|
||||
mega = backend.op(
|
||||
rank=rank,
|
||||
world_size=world,
|
||||
model_dim=model_dim,
|
||||
inter_dim=inter_dim,
|
||||
experts=experts,
|
||||
topk=topk,
|
||||
quant=quant,
|
||||
w1=layer._mega_w1,
|
||||
w1_scale=layer._mega_w1_scale,
|
||||
w2=layer._mega_w2,
|
||||
w2_scale=layer._mega_w2_scale,
|
||||
max_tok_per_rank=mtpr,
|
||||
swiglu_limit=swiglu_limit,
|
||||
)
|
||||
_MEGA_MOE_INSTANCE[key] = mega
|
||||
return mega
|
||||
|
||||
|
||||
def _swap_layer_weights(mega, layer) -> None:
|
||||
mega._s1_w1 = layer._mega_w1
|
||||
mega._s1_w1_scale = layer._mega_w1_scale
|
||||
mega.w2 = layer._mega_w2
|
||||
mega.w2_scale = layer._mega_w2_scale
|
||||
|
||||
|
||||
def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool:
|
||||
if not get_moe_a2a_backend().is_megamoe():
|
||||
return False
|
||||
if not getattr(moe.experts, "_mega_moe_weights_built", False):
|
||||
return False
|
||||
if get_is_capture_mode():
|
||||
return True
|
||||
global_num_tokens = get_dp_global_num_tokens()
|
||||
if global_num_tokens and not is_dsa_enable_prefill_cp():
|
||||
max_tokens = max(global_num_tokens)
|
||||
else:
|
||||
max_tokens = hidden_states.shape[0]
|
||||
return max_tokens <= _mtpr()
|
||||
|
||||
|
||||
def forward_mega_moe(
|
||||
moe: DeepseekV2MoE,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch | None = None,
|
||||
input_ids_global: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
overlap = _should_overlap_shared_and_routed(moe, num_tokens)
|
||||
if overlap:
|
||||
current_stream = torch.cuda.current_stream()
|
||||
moe.alt_stream.wait_stream(current_stream)
|
||||
shared_output = moe._forward_shared_experts(hidden_states)
|
||||
stream_context = torch.cuda.stream(moe.alt_stream)
|
||||
else:
|
||||
shared_output = moe._forward_shared_experts(hidden_states)
|
||||
stream_context = nullcontext()
|
||||
|
||||
with stream_context:
|
||||
output = _run_mega_routed(moe, hidden_states, forward_batch, input_ids_global)
|
||||
if overlap:
|
||||
current_stream.wait_stream(moe.alt_stream)
|
||||
if shared_output is not None:
|
||||
output.add_(shared_output)
|
||||
return output
|
||||
|
||||
|
||||
def _should_overlap_shared_and_routed(
|
||||
moe: DeepseekV2MoE,
|
||||
num_tokens: int,
|
||||
) -> bool:
|
||||
if envs.SGLANG_AITER_MEGA_RANK_SYNC.get():
|
||||
return False
|
||||
return (
|
||||
moe.alt_stream is not None
|
||||
and moe.num_fused_shared_experts == 0
|
||||
and num_tokens > 0
|
||||
and get_is_capture_mode()
|
||||
)
|
||||
|
||||
|
||||
def _run_mega_routed(
|
||||
moe: DeepseekV2MoE,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch | None,
|
||||
input_ids_global: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
num_tokens = hidden_states.shape[0]
|
||||
hidden_size = moe.config.hidden_size
|
||||
topk = moe.config.num_experts_per_tok + moe.num_fused_shared_experts
|
||||
experts = moe.experts.num_experts
|
||||
if num_tokens:
|
||||
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
|
||||
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
|
||||
topk_output = moe.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=(
|
||||
forward_batch.num_token_non_padded
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=moe.layer_id
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
x_in = hidden_states
|
||||
topk_ids = topk_output.topk_ids.to(torch.int32)
|
||||
topk_weights = topk_output.topk_weights.to(torch.float32)
|
||||
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
|
||||
_fill_padded_rows(
|
||||
topk_ids,
|
||||
forward_batch.num_token_non_padded,
|
||||
experts,
|
||||
)
|
||||
else:
|
||||
rank_sync = envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
|
||||
x_in = (
|
||||
hidden_states.new_ones((1, hidden_size))
|
||||
if rank_sync
|
||||
else hidden_states.new_zeros((1, hidden_size))
|
||||
)
|
||||
if rank_sync:
|
||||
ep_rank, ep_world = _ep_rank_world()
|
||||
expert_base = ep_rank * (experts // ep_world)
|
||||
topk_ids = (
|
||||
expert_base
|
||||
+ torch.arange(topk, device=hidden_states.device, dtype=torch.int32)
|
||||
).unsqueeze(0)
|
||||
topk_weights = hidden_states.new_full(
|
||||
(1, topk), 1.0 / topk, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
topk_ids = torch.full(
|
||||
(1, topk),
|
||||
experts,
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32)
|
||||
|
||||
selected_mtpr = _mtpr()
|
||||
x_in = x_in.contiguous()
|
||||
topk_ids = topk_ids.contiguous()
|
||||
topk_weights = topk_weights.contiguous()
|
||||
assert x_in.shape[0] <= selected_mtpr, (
|
||||
f"Aiter MegaMoE local tokens {x_in.shape[0]} exceed MTPR {selected_mtpr}"
|
||||
)
|
||||
mega = _get_or_build_mega_moe(
|
||||
moe.experts,
|
||||
model_dim=hidden_size,
|
||||
inter_dim=moe.config.moe_intermediate_size,
|
||||
experts=experts,
|
||||
topk=topk,
|
||||
quant=envs.SGLANG_AMD_FLYDSL_MEGA_QUANT.get() or "a8w4",
|
||||
swiglu_limit=float(getattr(moe.config, "swiglu_limit", 0.0) or 0.0),
|
||||
mtpr=selected_mtpr,
|
||||
)
|
||||
_swap_layer_weights(mega, moe.experts)
|
||||
|
||||
output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens]
|
||||
|
||||
from sglang.srt.models.deepseek_common.utils import _use_aiter
|
||||
|
||||
if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter):
|
||||
output.mul_(moe.routed_scaling_factor)
|
||||
return output
|
||||
@@ -99,6 +99,7 @@ from sglang.srt.eplb.expert_location_dispatch import (
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import is_allocation_symmetric
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
)
|
||||
@@ -2239,6 +2240,7 @@ def _post_process_topk_ids(
|
||||
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
|
||||
recorder_topk_ids = None
|
||||
_fold_pad_into_append = False
|
||||
recorder_was_fused = False
|
||||
if _is_cuda:
|
||||
# LP path: solve LP outside torch.compile (the solver contains an
|
||||
# EP all-reduce that can't run inside compiled regions).
|
||||
@@ -2302,10 +2304,34 @@ def _post_process_topk_ids(
|
||||
)
|
||||
if not _fold_pad_into_append:
|
||||
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0)
|
||||
if (
|
||||
_is_hip
|
||||
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
|
||||
and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get()
|
||||
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
|
||||
and layer_id is not None
|
||||
):
|
||||
from sglang.srt.eplb.eplb_map_record_fused import (
|
||||
eplb_map_and_record_fused,
|
||||
)
|
||||
|
||||
if get_moe_a2a_backend().is_megamoe():
|
||||
recorder = get_global_expert_distribution_recorder()
|
||||
load_buffer = recorder.get_current_pass_count_buffer(layer_id)
|
||||
if load_buffer is not None:
|
||||
fused_topk_ids = eplb_map_and_record_fused(
|
||||
topk_ids,
|
||||
expert_location_dispatch_info,
|
||||
load_buffer,
|
||||
num_token_non_padded,
|
||||
)
|
||||
if fused_topk_ids is not None:
|
||||
topk_ids = fused_topk_ids
|
||||
recorder_was_fused = True
|
||||
# The logical->physical remap is only meaningful when a real
|
||||
# expert-location mapping exists. With a trivial placement and EPLB off
|
||||
# the map is identity so the remap can be skipped safely.
|
||||
if _eplb_remap_enabled():
|
||||
if not recorder_was_fused and _eplb_remap_enabled():
|
||||
topk_ids = topk_ids_logical_to_physical(
|
||||
topk_ids, expert_location_dispatch_info
|
||||
)
|
||||
@@ -2314,7 +2340,7 @@ def _post_process_topk_ids(
|
||||
# That final pass re-zeros after any shared-expert append/remap, so a
|
||||
# second zeroing here would be redundant (zeroing is idempotent).
|
||||
|
||||
if recorder_topk_ids is None:
|
||||
if recorder_topk_ids is None and not recorder_was_fused:
|
||||
recorder_topk_ids = topk_ids
|
||||
|
||||
_aiter_append = num_fused_shared_experts > 0 and _use_aiter
|
||||
@@ -2712,9 +2738,10 @@ def select_experts(
|
||||
padded_rows_masked=padded_rows_masked,
|
||||
)
|
||||
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_topk_ids
|
||||
)
|
||||
if recorder_topk_ids is not None:
|
||||
get_global_expert_distribution_recorder().on_select_experts(
|
||||
topk_ids=recorder_topk_ids
|
||||
)
|
||||
|
||||
if packed_topk is not None:
|
||||
return StandardTopKOutputPacked(
|
||||
|
||||
@@ -1689,186 +1689,206 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
logger.warning_once("Dequantized FP4 MoE expert weights to FP8.")
|
||||
|
||||
def process_weights_after_loading_block_quant(self, layer: Module) -> None:
|
||||
# AMD FP4 experts: use aiter's native MXFP4 MoE path.
|
||||
# Skipped when dequant_fp4_to_fp8 is requested: this branch returns
|
||||
# unconditionally, so without the extra check SGLANG_DSV4_FP4_DEQUANT=1
|
||||
# is silently a no-op for routed experts on every AITER deployment.
|
||||
if _use_aiter and self.is_fp4_expert and not self.dequant_fp4_to_fp8:
|
||||
gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
|
||||
fp4_weight_dtype = _require_fp4_dtype()
|
||||
|
||||
# DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports
|
||||
# tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro
|
||||
# at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%.
|
||||
# shuffle_scale also supports non-256 shapes since aiter PR#4130.
|
||||
fp4_k_align = 128
|
||||
E, w13_N, w13_K_packed = layer.w13_weight.shape
|
||||
_, w2_N, w2_K_packed = layer.w2_weight.shape
|
||||
inter_per_part = w13_N // 2
|
||||
padded_inter = (
|
||||
(inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align
|
||||
)
|
||||
# Record the padding so fused_moe is told the real intermediate size
|
||||
# (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes
|
||||
# 128, SGLang previously defaulted to 0 -> computed the padded region).
|
||||
layer.intermediate_pad = padded_inter - inter_per_part
|
||||
layer.hidden_pad = 0
|
||||
if padded_inter != inter_per_part:
|
||||
pad_amount = padded_inter - inter_per_part
|
||||
fp4_block_k = 32
|
||||
|
||||
# Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed)
|
||||
old_w13 = layer.w13_weight.data
|
||||
new_w13 = torch.zeros(
|
||||
E,
|
||||
2 * padded_inter,
|
||||
w13_K_packed,
|
||||
dtype=old_w13.dtype,
|
||||
device=old_w13.device,
|
||||
)
|
||||
new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :]
|
||||
new_w13[:, padded_inter : padded_inter + inter_per_part, :] = old_w13[
|
||||
:, inter_per_part:, :
|
||||
]
|
||||
layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False)
|
||||
|
||||
# Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed)
|
||||
old_w2 = layer.w2_weight.data
|
||||
new_w2 = torch.zeros(
|
||||
E,
|
||||
w2_N,
|
||||
padded_inter // 2,
|
||||
dtype=old_w2.dtype,
|
||||
device=old_w2.device,
|
||||
)
|
||||
new_w2[:, :, :w2_K_packed] = old_w2
|
||||
layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False)
|
||||
|
||||
# Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k)
|
||||
old_s13 = layer.w13_weight_scale_inv.data
|
||||
_, _, s13_K = old_s13.shape
|
||||
new_s13 = torch.zeros(
|
||||
E,
|
||||
2 * padded_inter,
|
||||
s13_K,
|
||||
dtype=old_s13.dtype,
|
||||
device=old_s13.device,
|
||||
)
|
||||
new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :]
|
||||
new_s13[:, padded_inter : padded_inter + inter_per_part, :] = old_s13[
|
||||
:, inter_per_part:, :
|
||||
]
|
||||
layer.w13_weight_scale_inv = torch.nn.Parameter(
|
||||
new_s13, requires_grad=False
|
||||
if _use_aiter and self.is_fp4_expert:
|
||||
# aiter MegaMoEv2 builds from the packed FP4 layout, so it has to
|
||||
# claim the experts before the native-MXFP4 and dequant arms.
|
||||
if (
|
||||
get_moe_a2a_backend().is_megamoe()
|
||||
and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
|
||||
):
|
||||
if self.dequant_fp4_to_fp8:
|
||||
raise ValueError(
|
||||
"SGLANG_DSV4_FP4_DEQUANT strips the packed FP4 layout that "
|
||||
"aiter MegaMoEv2 builds from; unset it, or pick another "
|
||||
"--moe-a2a-backend."
|
||||
)
|
||||
from sglang.srt.layers.moe.mega_moe import (
|
||||
build_mega_moe_experts_weights,
|
||||
)
|
||||
|
||||
# Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k)
|
||||
old_s2 = layer.w2_weight_scale_inv.data
|
||||
new_s2 = torch.zeros(
|
||||
E,
|
||||
w2_N,
|
||||
padded_inter // fp4_block_k,
|
||||
dtype=old_s2.dtype,
|
||||
device=old_s2.device,
|
||||
fp4_weight_dtype = _require_fp4_dtype()
|
||||
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
|
||||
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
|
||||
build_mega_moe_experts_weights(layer)
|
||||
return
|
||||
|
||||
# AMD FP4 experts: use aiter's native MXFP4 MoE path.
|
||||
elif not self.dequant_fp4_to_fp8:
|
||||
gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
|
||||
fp4_weight_dtype = _require_fp4_dtype()
|
||||
|
||||
# DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports
|
||||
# tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro
|
||||
# at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%.
|
||||
# shuffle_scale also supports non-256 shapes since aiter PR#4130.
|
||||
fp4_k_align = 128
|
||||
E, w13_N, w13_K_packed = layer.w13_weight.shape
|
||||
_, w2_N, w2_K_packed = layer.w2_weight.shape
|
||||
inter_per_part = w13_N // 2
|
||||
padded_inter = (
|
||||
(inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align
|
||||
)
|
||||
new_s2[:, :, : old_s2.shape[2]] = old_s2
|
||||
layer.w2_weight_scale_inv = torch.nn.Parameter(
|
||||
new_s2, requires_grad=False
|
||||
)
|
||||
# Record the padding so fused_moe is told the real intermediate size
|
||||
# (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes
|
||||
# 128, SGLang previously defaulted to 0 -> computed the padded region).
|
||||
layer.intermediate_pad = padded_inter - inter_per_part
|
||||
layer.hidden_pad = 0
|
||||
if padded_inter != inter_per_part:
|
||||
pad_amount = padded_inter - inter_per_part
|
||||
fp4_block_k = 32
|
||||
|
||||
# Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed)
|
||||
old_w13 = layer.w13_weight.data
|
||||
new_w13 = torch.zeros(
|
||||
E,
|
||||
2 * padded_inter,
|
||||
w13_K_packed,
|
||||
dtype=old_w13.dtype,
|
||||
device=old_w13.device,
|
||||
)
|
||||
new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :]
|
||||
new_w13[:, padded_inter : padded_inter + inter_per_part, :] = (
|
||||
old_w13[:, inter_per_part:, :]
|
||||
)
|
||||
layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False)
|
||||
|
||||
# Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed)
|
||||
old_w2 = layer.w2_weight.data
|
||||
new_w2 = torch.zeros(
|
||||
E,
|
||||
w2_N,
|
||||
padded_inter // 2,
|
||||
dtype=old_w2.dtype,
|
||||
device=old_w2.device,
|
||||
)
|
||||
new_w2[:, :, :w2_K_packed] = old_w2
|
||||
layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False)
|
||||
|
||||
# Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k)
|
||||
old_s13 = layer.w13_weight_scale_inv.data
|
||||
_, _, s13_K = old_s13.shape
|
||||
new_s13 = torch.zeros(
|
||||
E,
|
||||
2 * padded_inter,
|
||||
s13_K,
|
||||
dtype=old_s13.dtype,
|
||||
device=old_s13.device,
|
||||
)
|
||||
new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :]
|
||||
new_s13[:, padded_inter : padded_inter + inter_per_part, :] = (
|
||||
old_s13[:, inter_per_part:, :]
|
||||
)
|
||||
layer.w13_weight_scale_inv = torch.nn.Parameter(
|
||||
new_s13, requires_grad=False
|
||||
)
|
||||
|
||||
# Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k)
|
||||
old_s2 = layer.w2_weight_scale_inv.data
|
||||
new_s2 = torch.zeros(
|
||||
E,
|
||||
w2_N,
|
||||
padded_inter // fp4_block_k,
|
||||
dtype=old_s2.dtype,
|
||||
device=old_s2.device,
|
||||
)
|
||||
new_s2[:, :, : old_s2.shape[2]] = old_s2
|
||||
layer.w2_weight_scale_inv = torch.nn.Parameter(
|
||||
new_s2, requires_grad=False
|
||||
)
|
||||
|
||||
for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"):
|
||||
scale = getattr(layer, scale_name)
|
||||
num_experts, num_rows, _ = scale.shape
|
||||
is_w13_scale = scale_name == "w13_weight_scale_inv"
|
||||
if _is_gfx1250_supported:
|
||||
scale.data = moe_shuffle_scale(
|
||||
scale.contiguous(),
|
||||
experts_cnt=num_experts,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=is_w13_scale,
|
||||
)
|
||||
else:
|
||||
scale_2d = scale.reshape(-1, scale.shape[-1])
|
||||
scale.data = shuffle_scale(
|
||||
scale_2d, num_experts, gu_intv, is_w13_scale
|
||||
)
|
||||
|
||||
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
|
||||
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
|
||||
|
||||
for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"):
|
||||
scale = getattr(layer, scale_name)
|
||||
num_experts, num_rows, _ = scale.shape
|
||||
is_w13_scale = scale_name == "w13_weight_scale_inv"
|
||||
if _is_gfx1250_supported:
|
||||
scale.data = moe_shuffle_scale(
|
||||
scale.contiguous(),
|
||||
experts_cnt=num_experts,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=is_w13_scale,
|
||||
)
|
||||
else:
|
||||
scale_2d = scale.reshape(-1, scale.shape[-1])
|
||||
scale.data = shuffle_scale(
|
||||
scale_2d, num_experts, gu_intv, is_w13_scale
|
||||
)
|
||||
|
||||
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
|
||||
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
|
||||
|
||||
if _is_gfx1250_supported:
|
||||
is_shuffled = True
|
||||
layer.w13_weight.data = moe_shuffle_weight(
|
||||
layer.w13_weight,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=True,
|
||||
)
|
||||
layer.w2_weight.data = moe_shuffle_weight(
|
||||
layer.w2_weight,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=False,
|
||||
)
|
||||
else:
|
||||
is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4
|
||||
if is_shuffled:
|
||||
shuffle_gu_intv = gu_intv and not _use_aiter_a8w4
|
||||
layer.w13_weight.data = shuffle_weight(
|
||||
is_shuffled = True
|
||||
layer.w13_weight.data = moe_shuffle_weight(
|
||||
layer.w13_weight,
|
||||
is_guinterleave=shuffle_gu_intv,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=True,
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight(
|
||||
layer.w2_weight.data = moe_shuffle_weight(
|
||||
layer.w2_weight,
|
||||
is_guinterleave=shuffle_gu_intv,
|
||||
is_guinterleave=gu_intv,
|
||||
gate_up=False,
|
||||
)
|
||||
layer.w13_weight.is_shuffled = is_shuffled
|
||||
layer.w2_weight.is_shuffled = is_shuffled
|
||||
return
|
||||
else:
|
||||
is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4
|
||||
if is_shuffled:
|
||||
shuffle_gu_intv = gu_intv and not _use_aiter_a8w4
|
||||
layer.w13_weight.data = shuffle_weight(
|
||||
layer.w13_weight,
|
||||
is_guinterleave=shuffle_gu_intv,
|
||||
gate_up=True,
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight(
|
||||
layer.w2_weight,
|
||||
is_guinterleave=shuffle_gu_intv,
|
||||
gate_up=False,
|
||||
)
|
||||
layer.w13_weight.is_shuffled = is_shuffled
|
||||
layer.w2_weight.is_shuffled = is_shuffled
|
||||
return
|
||||
|
||||
# ROCm AITER: bypass the native FP4 early return when DSV4 dequant is
|
||||
# requested, then use the standard block-FP8 MoE path.
|
||||
if self.is_fp4_expert and self.dequant_fp4_to_fp8 and _use_aiter:
|
||||
self._dequantize_aiter_fp4_experts(layer)
|
||||
self.weight_block_size = [128, 128]
|
||||
# DSV4 dequant requested: convert to block-FP8 and use the standard
|
||||
# block-FP8 MoE path.
|
||||
else:
|
||||
self._dequantize_aiter_fp4_experts(layer)
|
||||
self.weight_block_size = [128, 128]
|
||||
|
||||
# gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn.
|
||||
if _is_fp8_fnuz:
|
||||
w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=layer.w13_weight,
|
||||
weight_scale=layer.w13_weight_scale_inv,
|
||||
input_scale=None,
|
||||
)
|
||||
w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=layer.w2_weight,
|
||||
weight_scale=layer.w2_weight_scale_inv,
|
||||
input_scale=None,
|
||||
)
|
||||
layer.w13_weight = Parameter(w13_weight, requires_grad=False)
|
||||
layer.w13_weight_scale_inv = Parameter(
|
||||
w13_weight_scale, requires_grad=False
|
||||
)
|
||||
layer.w2_weight = Parameter(w2_weight, requires_grad=False)
|
||||
layer.w2_weight_scale_inv = Parameter(
|
||||
w2_weight_scale, requires_grad=False
|
||||
)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
# gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn.
|
||||
if _is_fp8_fnuz:
|
||||
w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=layer.w13_weight,
|
||||
weight_scale=layer.w13_weight_scale_inv,
|
||||
input_scale=None,
|
||||
)
|
||||
w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=layer.w2_weight,
|
||||
weight_scale=layer.w2_weight_scale_inv,
|
||||
input_scale=None,
|
||||
)
|
||||
layer.w13_weight = Parameter(w13_weight, requires_grad=False)
|
||||
layer.w13_weight_scale_inv = Parameter(
|
||||
w13_weight_scale, requires_grad=False
|
||||
)
|
||||
layer.w2_weight = Parameter(w2_weight, requires_grad=False)
|
||||
layer.w2_weight_scale_inv = Parameter(
|
||||
w2_weight_scale, requires_grad=False
|
||||
)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
|
||||
# consumes un-shuffled weights (shuffling the wrong runner corrupts output).
|
||||
runner_is_aiter = (
|
||||
getattr(self, "runner", None) is not None
|
||||
and self.runner.runner_backend.is_aiter()
|
||||
)
|
||||
if _use_aiter and runner_is_aiter:
|
||||
layer.w13_weight.data = shuffle_weight(
|
||||
layer.w13_weight.contiguous(), (16, 16)
|
||||
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
|
||||
# consumes un-shuffled weights (shuffling the wrong runner corrupts output).
|
||||
runner_is_aiter = (
|
||||
getattr(self, "runner", None) is not None
|
||||
and self.runner.runner_backend.is_aiter()
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight(
|
||||
layer.w2_weight.contiguous(), (16, 16)
|
||||
)
|
||||
return
|
||||
if _use_aiter and runner_is_aiter:
|
||||
layer.w13_weight.data = shuffle_weight(
|
||||
layer.w13_weight.contiguous(), (16, 16)
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight(
|
||||
layer.w2_weight.contiguous(), (16, 16)
|
||||
)
|
||||
return
|
||||
|
||||
if self.convert_mxfp8_to_block:
|
||||
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.utils.common import (
|
||||
Range,
|
||||
ceil_align,
|
||||
flatten_arrays_to_pinned_cpu,
|
||||
is_hip,
|
||||
is_pin_memory_available,
|
||||
)
|
||||
from sglang.srt.utils.weight_versions import (
|
||||
@@ -163,6 +164,8 @@ _MM_HASH_MASK = (1 << 64) - 1
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
ReturnHiddenStatesMode = Union[bool, Literal["last"]]
|
||||
SamplingLogprobsMode = Literal["selected", "support"]
|
||||
@@ -2508,6 +2511,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# For DP attention
|
||||
global_num_tokens: Optional[List[int]] = None
|
||||
global_num_tokens_for_logprob: Optional[List[int]] = None
|
||||
# Full DP token vector retained for Aiter MegaMoE even when the normal MLP
|
||||
# TP gather path stores only this rank's token count.
|
||||
global_spec_verify_tier_num_tokens: Optional[List[int]] = None
|
||||
|
||||
# Member rows riding one forward; None whenever reqs and rows are 1:1.
|
||||
@@ -3839,6 +3844,27 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
extend_num_tokens=self.extend_num_tokens,
|
||||
)
|
||||
|
||||
def _swa_eviction_trigger(self, sliding_window_size: int, eviction_interval: int):
|
||||
"""Pick this forward's per-request SWA eviction trigger.
|
||||
|
||||
Default: evict only once >= eviction_interval tokens have slid out of
|
||||
the window, amortizing eviction work while keeping each request's
|
||||
overshoot within the interval the pool budget reserves. Gating on
|
||||
accumulated tokens rather than an iteration-counter phase cannot
|
||||
starve, because seqlen progress is monotonic per KV handle.
|
||||
|
||||
Aiter MegaMoE DSV4 keeps the older forward-interval cadence instead:
|
||||
per-request token gating synchronizes SWA pressure across DP ranks and
|
||||
triggers a retraction / re-prefill storm at high concurrency.
|
||||
"""
|
||||
if _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get():
|
||||
due_this_forward = (self.forward_iter or 0) % eviction_interval == 0
|
||||
return lambda req: due_this_forward
|
||||
return lambda req: (
|
||||
req.seqlen - 1 - sliding_window_size
|
||||
>= req.kv.swa_evicted_seqlen + eviction_interval
|
||||
)
|
||||
|
||||
def maybe_evict_swa(self):
|
||||
if self.tree_cache.supports_swa():
|
||||
sliding_window_size = self.tree_cache.sliding_window_size
|
||||
@@ -3849,22 +3875,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
|
||||
swa_evict_due = self._swa_eviction_trigger(
|
||||
sliding_window_size, eviction_interval
|
||||
)
|
||||
self.token_to_kv_pool_allocator.free_group_begin()
|
||||
for idx, req in enumerate(self.reqs):
|
||||
if self.forward_mode.is_decode():
|
||||
# We set evict_swa condition here with two reasons:
|
||||
# 1. In overlap scheduler, we cannot evict swa when req.decode_batch_idx == 0 since the prev extend batch is still running.
|
||||
# 2. Evict only once >= eviction_interval tokens have slid
|
||||
# out of the window, amortizing eviction work while keeping
|
||||
# each request's overshoot within the interval the pool
|
||||
# budget reserves. Gating on accumulated tokens (rather
|
||||
# than an iteration-counter phase) cannot starve because
|
||||
# seqlen progress is monotonic per KV handle.
|
||||
# In overlap scheduler, we cannot evict swa when
|
||||
# req.decode_batch_idx == 0 since the prev extend batch is
|
||||
# still running. `_swa_eviction_trigger` owns the rest.
|
||||
if (
|
||||
req.decode_batch_idx >= 1
|
||||
and req.kv.holds_kv
|
||||
and req.seqlen - 1 - sliding_window_size
|
||||
>= req.kv.swa_evicted_seqlen + eviction_interval
|
||||
and swa_evict_due(req)
|
||||
):
|
||||
self._evict_swa(req, req.seqlen - 1)
|
||||
|
||||
|
||||
@@ -226,7 +226,6 @@ def _update_gather_batch(
|
||||
require_mlp_tp_gather: bool,
|
||||
skip_global_metadata=False,
|
||||
):
|
||||
# TODO: handle the case when moe_dense_tp_size != 1
|
||||
if not require_mlp_tp_gather:
|
||||
batch.global_num_tokens = [mlp_sync_info.num_tokens]
|
||||
batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob]
|
||||
@@ -493,7 +492,6 @@ def prepare_mlp_sync_batch_raw(
|
||||
mlp_sync_info.tp0_info_cpu[:, 4:6],
|
||||
)
|
||||
)
|
||||
|
||||
# Decide whether to emit idle batch
|
||||
if skip_all_gather:
|
||||
# Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently)
|
||||
|
||||
@@ -85,6 +85,7 @@ _skip_attn_backend_init_warned = False
|
||||
|
||||
_is_npu = is_npu()
|
||||
_is_cpu = is_cpu()
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
def _build_forward_token_modalities(
|
||||
@@ -155,6 +156,21 @@ def _maybe_build_forward_token_modalities(
|
||||
)
|
||||
|
||||
|
||||
def _mega_moe_materializes_idle_rank(batch: ForwardBatch) -> bool:
|
||||
"""Whether aiter MegaMoE needs this idle rank to carry a token anyway.
|
||||
|
||||
MegaMoE's dispatch is rank-synchronous: when the batch is globally a
|
||||
prefill, a rank with nothing to do still has to enter the collective, so it
|
||||
runs a fabricated one-token extend instead of sitting the forward out.
|
||||
"""
|
||||
return bool(
|
||||
_is_hip
|
||||
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
|
||||
and batch.is_extend_in_batch
|
||||
and batch.forward_mode.is_idle()
|
||||
)
|
||||
|
||||
|
||||
def _elastic_should_preserve_local_token_counts(
|
||||
*,
|
||||
model_runner: ModelRunner,
|
||||
@@ -1540,6 +1556,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
|
||||
self._original_batch_size = self.batch_size
|
||||
global_num_tokens = list(self.global_num_tokens_cpu)
|
||||
# MegaMoEv2 dispatch is rank-synchronous, so an idle rank still carries the
|
||||
# one row _run_mega_routed fabricates for it. HIP-only, False elsewhere.
|
||||
mega_moe_idle_materialize = _mega_moe_materializes_idle_rank(self)
|
||||
if mega_moe_idle_materialize:
|
||||
global_num_tokens = [1] * len(global_num_tokens)
|
||||
sync_group_size = len(global_num_tokens)
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
@@ -1582,6 +1603,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# Preserve each rank's forward mode on coordinated steps.
|
||||
dp_padding_mode = DpPaddingMode.SUM_LEN
|
||||
self.dp_padding_mode = dp_padding_mode
|
||||
# Read once here, where dp_padding_mode is final: the idle-row branch
|
||||
# below runs after another arm may have rewritten self.forward_mode.
|
||||
materializes_dummy_extend = mega_moe_idle_materialize or (
|
||||
self.is_extend_in_batch and dp_padding_mode.is_max_len()
|
||||
)
|
||||
|
||||
if dp_padding_mode.is_max_len():
|
||||
# when DP gather mode is all gather, we will use
|
||||
@@ -1634,7 +1660,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.forward_mode = ForwardMode.TARGET_VERIFY
|
||||
# Invert the spec_scale_global_num_tokens scaling.
|
||||
bs = self.batch_size = num_tokens // self.spec_info.num_tokens_per_req
|
||||
elif self.is_extend_in_batch and dp_padding_mode.is_max_len():
|
||||
elif materializes_dummy_extend:
|
||||
self._original_forward_mode = self.forward_mode
|
||||
self.forward_mode = ForwardMode.EXTEND
|
||||
# Fabricate a single dummy request covering num_tokens for an
|
||||
|
||||
@@ -54,6 +54,7 @@ from sglang.srt.eplb.expert_distribution import (
|
||||
ExpertDistributionRecorder,
|
||||
get_global_expert_distribution_recorder,
|
||||
set_global_expert_distribution_recorder,
|
||||
should_advance_eplb_counter,
|
||||
)
|
||||
from sglang.srt.eplb.expert_location import (
|
||||
ExpertLocationMetadata,
|
||||
@@ -1767,7 +1768,8 @@ class ModelRunner:
|
||||
no_copy_to_cpu=no_copy_to_cpu,
|
||||
)
|
||||
|
||||
if self.eplb_manager is not None:
|
||||
# should_advance_eplb_counter is always True on non-hip platform
|
||||
if self.eplb_manager is not None and should_advance_eplb_counter(forward_batch):
|
||||
self.eplb_manager.on_forward_pass_end()
|
||||
|
||||
if dumper.may_enable:
|
||||
|
||||
Reference in New Issue
Block a user