diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index d10ec4e527..d45ae3caff 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -36,11 +36,15 @@ def check_mega_moe_compat(server_args: ServerArgs) -> None: "--enable-single-batch-overlap." ) platform = get_platform() - if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)): + if not ( + (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)) + or platform.is_hip + ): raise ValueError( "--moe-a2a-backend megamoe needs a CUDA SM90 GPU (block-FP8 experts) " "or an SM100-class GPU (MXFP4 / NVFP4 experts); it runs DeepGEMM " - "kernels over CUDA symmetric memory." + "kernels over CUDA symmetric memory. On ROCm, set " + "SGLANG_AMD_USE_FLYDSL_MEGA_MOE=1 to select aiter MegaMoEv2 instead." ) @@ -98,6 +102,8 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) -> cfg = resolving_view(server_args) if cfg.moe_a2a_backend != "megamoe": return + if get_platform().is_hip: + return max_tokens_per_rank = ( envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get() diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 890d889648..381a52e539 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -909,6 +909,12 @@ class Envs: SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256) SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang") SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False) + SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False) + SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192) + SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("") + SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False) + SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False) + SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD = EnvBool(False) # Above 8192 tokens of context, aiter's non-static workspace is large enough # that mem_fraction_static is scaled by 0.85 to leave room for it. Set this to # honor an explicitly passed --mem-fraction-static instead. Off by default: diff --git a/python/sglang/srt/eplb/eplb_map_record_fused.py b/python/sglang/srt/eplb/eplb_map_record_fused.py new file mode 100644 index 0000000000..62ab8ec7c3 --- /dev/null +++ b/python/sglang/srt/eplb/eplb_map_record_fused.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from typing import Optional + +import torch + +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo + +try: + import triton + import triton.language as tl + + _HAS_TRITON = True +except ImportError: + _HAS_TRITON = False + + +if _HAS_TRITON: + + @triton.jit + def _eplb_map_record_hist_kernel( + topk_ids_ptr, + dispatch_ptr, + out_ids_ptr, + load_ptr, + num_valid_tokens_ptr, + num_logical, + num_physical, + numel, + HAS_DISPATCH: tl.constexpr, + HAS_VALID_COUNT: tl.constexpr, + BLOCK: tl.constexpr, + NUM_BINS: tl.constexpr, + TOP_K: tl.constexpr, + ): + offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = offs < numel + logical_id = tl.load(topk_ids_ptr + offs, mask=mask, other=-1).to(tl.int64) + valid_logical = (logical_id >= 0) & (logical_id < num_logical) + safe_logical = tl.where(valid_logical, logical_id, 0) + if HAS_DISPATCH: + physical_id = tl.load( + dispatch_ptr + safe_logical, + mask=mask & valid_logical, + other=0, + ).to(tl.int64) + else: + physical_id = logical_id + output_id = tl.where(valid_logical, physical_id, logical_id) + tl.store(out_ids_ptr + offs, output_id, mask=mask) + + valid_row = mask + if HAS_VALID_COUNT: + num_valid_tokens = tl.load(num_valid_tokens_ptr).to(tl.int64) + valid_row &= (offs // TOP_K) < num_valid_tokens + in_range = ( + valid_row + & valid_logical + & (physical_id >= 0) + & (physical_id < num_physical) + ) + bin_idx = tl.where(in_range, physical_id, num_physical).to(tl.int32) + histogram = tl.histogram(bin_idx, NUM_BINS) + bins = tl.arange(0, NUM_BINS) + tl.atomic_add(load_ptr + bins, histogram, mask=bins < num_physical) + + +def eplb_map_and_record_fused( + topk_ids: torch.Tensor, + dispatch_info: Optional[ExpertLocationDispatchInfo], + load_buffer: torch.Tensor, + num_token_non_padded: Optional[torch.Tensor], +) -> Optional[torch.Tensor]: + if not _HAS_TRITON or not topk_ids.is_cuda or topk_ids.numel() == 0: + return None + if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm != "static": + return None + + dispatch = ( + dispatch_info.partial_logical_to_rank_dispatch_physical_map + if dispatch_info is not None + else None + ) + if dispatch_info is not None and dispatch is None: + return None + num_physical = int(load_buffer.numel()) + num_logical = int(dispatch.numel()) if dispatch is not None else num_physical + top_k = int(topk_ids.shape[-1]) + numel = topk_ids.numel() + output = torch.empty_like(topk_ids) + dispatch_ptr = dispatch if dispatch is not None else topk_ids + valid_count_ptr = ( + num_token_non_padded if num_token_non_padded is not None else topk_ids + ) + grid = (triton.cdiv(numel, 256),) + _eplb_map_record_hist_kernel[grid]( + topk_ids.contiguous(), + dispatch_ptr, + output, + load_buffer, + valid_count_ptr, + num_logical, + num_physical, + numel, + HAS_DISPATCH=dispatch is not None, + HAS_VALID_COUNT=num_token_non_padded is not None, + BLOCK=256, + NUM_BINS=1 << num_physical.bit_length(), + TOP_K=top_k, + ) + return output + + +__all__ = ["eplb_map_and_record_fused"] diff --git a/python/sglang/srt/eplb/expert_distribution.py b/python/sglang/srt/eplb/expert_distribution.py index 05f5a0e223..4f99f17b79 100644 --- a/python/sglang/srt/eplb/expert_distribution.py +++ b/python/sglang/srt/eplb/expert_distribution.py @@ -39,7 +39,7 @@ import torch import torch.distributed from sglang.srt.environ import envs -from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.observability.metrics_collector import ( STAT_LOGGER_ROLE_EXPERT_DISPATCH, ExpertDispatchCollector, @@ -48,6 +48,7 @@ from sglang.srt.observability.metrics_collector import ( from sglang.srt.runtime_context import get_device as get_device_namespace from sglang.srt.runtime_context import ( get_exec, + get_platform, get_schedule, logs_expert_balancedness_to_server_log, reports_expert_balancedness, @@ -59,6 +60,57 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) + +# AMD specific function +def megamoe_prefill_only_recorder_enabled() -> bool: + # The prefill-only arm belongs to aiter MegaMoEv2, which only exists on ROCm. + # Read the backend through the resolved accessor, not off a supplied + # ServerArgs: test_supplied_instance_exposure_ratchet pins (file, field) + # pairs where a `server_args` parameter is read for a field that resolution + # fills in, and that surface may only shrink. + from sglang.srt.layers.moe import get_moe_a2a_backend + + return ( + get_platform().is_hip + and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get() + and envs.SGLANG_AITER_MEGA_RANK_SYNC.get() + and get_moe_a2a_backend().is_megamoe() + ) + + +# non-hip platform always return True +def should_record_forward_pass(forward_batch: ForwardBatch) -> bool: + if not megamoe_prefill_only_recorder_enabled(): + return True + if _is_model_capture_mode(): + return False + if forward_batch.forward_mode not in ( + ForwardMode.EXTEND, + ForwardMode.SPLIT_PREFILL, + ): + return False + return int(forward_batch.extend_num_tokens or 0) > 0 + + +# non-hip platform always return True +def should_advance_eplb_counter(forward_batch: ForwardBatch) -> bool: + """Whether a finished pass counts towards the EPLB rebalance interval. + + Unlike `should_record_forward_pass`, which is local to this rank, the + counter has to advance in lockstep across ranks, so it keys off the globally + synchronized `is_extend_in_batch` rather than this rank's forward mode. + """ + if not megamoe_prefill_only_recorder_enabled(): + return True + return forward_batch.is_extend_in_batch + + +def _is_model_capture_mode() -> bool: + from sglang.srt.model_executor.runner_utils.capture_mode import is_capture_mode + + return is_capture_mode + + # --------------------------------------- Entrypoint ----------------------------------------- _OutputMode = Literal["file", "object"] @@ -117,6 +169,9 @@ class ExpertDistributionRecorder(ABC): def on_select_experts(self, topk_ids: torch.Tensor): pass + def get_current_pass_count_buffer(self, layer_idx: int): + return None + def on_deepep_dispatch_normal( self, local_physical_count_of_layer: List[int], @@ -164,6 +219,7 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder): self._recording = False self._disable_all = False + self._record_current_pass = False self._current_forward_pass_id = Withable() self._current_layer_idx = Withable() self._current_debug_name = Withable() @@ -208,13 +264,17 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder): def _on_forward_pass_start(self, forward_batch: ForwardBatch): if not self._recording: + self._record_current_pass = False + return + self._record_current_pass = should_record_forward_pass(forward_batch) + if not self._record_current_pass: return for gatherer_key, gatherer in self._single_pass_gatherers.items(): gatherer.reset() gatherer.on_forward_pass_start(forward_batch) def _on_forward_pass_end(self, forward_pass_id: int, outputs: Dict[str, Any]): - if not self._recording: + if not self._recording or not self._record_current_pass: return for gatherer_key, gatherer in self._single_pass_gatherers.items(): single_pass_data = gatherer.collect() @@ -225,6 +285,14 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder): def on_select_experts(self, topk_ids: torch.Tensor): self._on_hook("on_select_experts", topk_ids=topk_ids) + def get_current_pass_count_buffer(self, layer_idx: int): + if not self._recording or not self._record_current_pass: + return None + gatherer = self._single_pass_gatherers.get(_SINGLE_PASS_GATHERER_KEY_PRIMARY) + if not isinstance(gatherer, _SelectExpertsSinglePassGatherer): + return None + return gatherer._data[layer_idx] + def on_deepep_dispatch_normal( self, local_physical_count_of_layer: List[int], @@ -251,6 +319,8 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder): def _on_hook(self, hook_name: str, **kwargs): if self._disable_all: return + if megamoe_prefill_only_recorder_enabled() and not self._record_current_pass: + return if not ( self._recording or torch.get_device_module().is_current_stream_capturing() ): @@ -548,7 +618,9 @@ class _SelectExpertsSinglePassGatherer(_LayerBasedGpuSinglePassGatherer): # can optimize (e.g. fuse / compile) def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor): topk_ids = topk_ids.flatten() - mask = topk_ids != -1 + mask = (topk_ids >= 0) & ( + topk_ids < self._expert_location_metadata.num_physical_experts + ) self._data[layer_idx, :].scatter_add_( dim=0, index=topk_ids.masked_fill(~mask, 0).long(), src=mask.int() ) diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 31e481aed1..2478611b14 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -34,6 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.models.deepseek_common.utils import _device_sm from sglang.srt.runtime_context import get_exec +from sglang.srt.utils import is_hip if TYPE_CHECKING: from deep_gemm import SymmBuffer @@ -43,6 +44,13 @@ if TYPE_CHECKING: _MEGA_MOE_SYMM_BUFFER: dict = {} +_is_hip = is_hip() + + +def _use_amd_flydsl_mega_moe() -> bool: + # aiter MegaMoEv2 exists only on ROCm; the platform check keeps the env from + # diverting a CUDA run into a path whose kernels it does not have. + return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get() def _mega_moe_mma_type(experts=None) -> str: @@ -152,6 +160,13 @@ def is_mega_moe_experts_ready(experts) -> bool: def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool: + if _use_amd_flydsl_mega_moe(): + from sglang.srt.layers.moe.mega_moe_flydsl import ( + should_use_mega_moe as should_use_flydsl_mega_moe, + ) + + return should_use_flydsl_mega_moe(moe, hidden_states) + if not get_moe_a2a_backend().is_megamoe(): return False if not is_mega_moe_experts_ready(moe.experts): @@ -174,6 +189,15 @@ def forward_mega_moe( forward_batch: Optional[ForwardBatch] = None, input_ids_global: Optional[torch.Tensor] = None, ) -> torch.Tensor: + if _use_amd_flydsl_mega_moe(): + from sglang.srt.layers.moe.mega_moe_flydsl import ( + forward_mega_moe as forward_flydsl_mega_moe, + ) + + return forward_flydsl_mega_moe( + moe, hidden_states, forward_batch, input_ids_global + ) + num_tokens = hidden_states.shape[0] sbo_overlap_flag = ( @@ -432,6 +456,14 @@ def _transpose_mega_moe_sf_for_utccp(sf: torch.Tensor) -> torch.Tensor: def build_mega_moe_experts_weights(experts) -> None: + if _use_amd_flydsl_mega_moe(): + from sglang.srt.layers.moe.mega_moe_flydsl import ( + build_mega_moe_experts_weights as build_flydsl_mega_moe_weights, + ) + + build_flydsl_mega_moe_weights(experts) + return + from deep_gemm import ( transform_sf_into_required_layout, ) diff --git a/python/sglang/srt/layers/moe/mega_moe_flydsl.py b/python/sglang/srt/layers/moe/mega_moe_flydsl.py new file mode 100644 index 0000000000..0990258cee --- /dev/null +++ b/python/sglang/srt/layers/moe/mega_moe_flydsl.py @@ -0,0 +1,316 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0. +"""Aiter MegaMoEV2 adapter for the SGLang MegaMoE hooks.""" + +from __future__ import annotations + +from contextlib import nullcontext +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows +from sglang.srt.environ import envs +from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo +from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp +from sglang.srt.layers.dp_attention import get_dp_global_num_tokens +from sglang.srt.layers.moe.utils import get_moe_a2a_backend +from sglang.srt.model_executor.runner import get_is_capture_mode +from sglang.srt.runtime_context import get_parallel + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + from sglang.srt.models.deepseek_v2 import DeepseekV2MoE + + +_MORI_SHMEM_READY = False +_MEGA_MOE_INSTANCE: dict = {} + + +def _import_aiter_megamoe(): + try: + from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2 + from aiter.ops.shuffle import shuffle_scale_a16w4, shuffle_weight_a16w4 + except ImportError as exc: + raise ImportError( + "Aiter MegaMoE requires a package containing MegaMoEV2 and the " + f"A16W4 shuffle operators. Original import error: {exc}" + ) from exc + + return SimpleNamespace( + op=MegaMoEV2, + shuffle_weight=shuffle_weight_a16w4, + shuffle_scale=shuffle_scale_a16w4, + ) + + +def _mtpr() -> int: + mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get()) + if mtpr <= 0 or mtpr & (mtpr - 1): + raise ValueError( + f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr} must be a positive power of two" + ) + return mtpr + + +def _ep_rank_world(): + parallel = get_parallel() + return parallel.moe_ep_rank, parallel.moe_ep_size + + +def _ensure_mori_shmem() -> None: + global _MORI_SHMEM_READY + if _MORI_SHMEM_READY: + return + + import mori.shmem + + group_name = "megamoe_aiter" + cpu_group = get_parallel().moe_ep_group.cpu_group + try: + torch._C._distributed_c10d._register_process_group(group_name, cpu_group) + except Exception as exc: + if "already registered" not in str(exc): + raise + mori.shmem.shmem_torch_process_group_init(group_name) + _MORI_SHMEM_READY = True + + +def build_mega_moe_experts_weights(layer) -> None: + if getattr(layer, "_mega_moe_weights_built", False): + return + + backend = _import_aiter_megamoe() + + def scale_param(name): + scale = getattr(layer, name, None) + if scale is None: + scale = getattr(layer, name + "_inv", None) + if scale is None: + raise AttributeError(f"Aiter MegaMoE weight scale {name} is missing") + return scale + + def shuffle_existing(weight, scale, *, gate_up): + experts = weight.shape[0] + shuffled_weight = backend.shuffle_weight(weight, 16, gate_up).contiguous() + shuffled_scale = backend.shuffle_scale( + scale.view(-1, scale.shape[-1]), experts, gate_up + ).contiguous() + return ( + shuffled_weight.view(torch.uint8).contiguous(), + shuffled_scale.view(torch.uint8).contiguous(), + ) + + w13_scale = scale_param("w13_weight_scale") + w2_scale = scale_param("w2_weight_scale") + layer._mega_w1, layer._mega_w1_scale = shuffle_existing( + layer.w13_weight.data, w13_scale.data, gate_up=True + ) + layer._mega_w2, layer._mega_w2_scale = shuffle_existing( + layer.w2_weight.data, w2_scale.data, gate_up=False + ) + + experts = layer.w13_weight.shape[0] + layer.w13_weight.data = layer._mega_w1.view(experts, -1) + w13_scale.data = layer._mega_w1_scale.view(experts, -1) + layer.w2_weight.data = layer._mega_w2.view(experts, -1) + w2_scale.data = layer._mega_w2_scale.view(experts, -1) + layer._mega_moe_weights_built = True + + +def _get_or_build_mega_moe( + layer, + *, + model_dim: int, + inter_dim: int, + experts: int, + topk: int, + quant: str, + swiglu_limit: float, + mtpr: int, +): + _ensure_mori_shmem() + backend = _import_aiter_megamoe() + rank, world = _ep_rank_world() + key = ( + rank, + world, + model_dim, + inter_dim, + experts, + topk, + quant, + swiglu_limit, + mtpr, + ) + mega = _MEGA_MOE_INSTANCE.get(key) + if mega is None: + mega = backend.op( + rank=rank, + world_size=world, + model_dim=model_dim, + inter_dim=inter_dim, + experts=experts, + topk=topk, + quant=quant, + w1=layer._mega_w1, + w1_scale=layer._mega_w1_scale, + w2=layer._mega_w2, + w2_scale=layer._mega_w2_scale, + max_tok_per_rank=mtpr, + swiglu_limit=swiglu_limit, + ) + _MEGA_MOE_INSTANCE[key] = mega + return mega + + +def _swap_layer_weights(mega, layer) -> None: + mega._s1_w1 = layer._mega_w1 + mega._s1_w1_scale = layer._mega_w1_scale + mega.w2 = layer._mega_w2 + mega.w2_scale = layer._mega_w2_scale + + +def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool: + if not get_moe_a2a_backend().is_megamoe(): + return False + if not getattr(moe.experts, "_mega_moe_weights_built", False): + return False + if get_is_capture_mode(): + return True + global_num_tokens = get_dp_global_num_tokens() + if global_num_tokens and not is_dsa_enable_prefill_cp(): + max_tokens = max(global_num_tokens) + else: + max_tokens = hidden_states.shape[0] + return max_tokens <= _mtpr() + + +def forward_mega_moe( + moe: DeepseekV2MoE, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch | None = None, + input_ids_global: torch.Tensor | None = None, +) -> torch.Tensor: + num_tokens = hidden_states.shape[0] + overlap = _should_overlap_shared_and_routed(moe, num_tokens) + if overlap: + current_stream = torch.cuda.current_stream() + moe.alt_stream.wait_stream(current_stream) + shared_output = moe._forward_shared_experts(hidden_states) + stream_context = torch.cuda.stream(moe.alt_stream) + else: + shared_output = moe._forward_shared_experts(hidden_states) + stream_context = nullcontext() + + with stream_context: + output = _run_mega_routed(moe, hidden_states, forward_batch, input_ids_global) + if overlap: + current_stream.wait_stream(moe.alt_stream) + if shared_output is not None: + output.add_(shared_output) + return output + + +def _should_overlap_shared_and_routed( + moe: DeepseekV2MoE, + num_tokens: int, +) -> bool: + if envs.SGLANG_AITER_MEGA_RANK_SYNC.get(): + return False + return ( + moe.alt_stream is not None + and moe.num_fused_shared_experts == 0 + and num_tokens > 0 + and get_is_capture_mode() + ) + + +def _run_mega_routed( + moe: DeepseekV2MoE, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch | None, + input_ids_global: torch.Tensor | None, +) -> torch.Tensor: + num_tokens = hidden_states.shape[0] + hidden_size = moe.config.hidden_size + topk = moe.config.num_experts_per_tok + moe.num_fused_shared_experts + experts = moe.experts.num_experts + if num_tokens: + router_logits = moe.gate(hidden_states, forward_batch=forward_batch) + topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {} + topk_output = moe.topk( + hidden_states, + router_logits, + num_token_non_padded=( + forward_batch.num_token_non_padded + if forward_batch is not None + else None + ), + expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( + layer_id=moe.layer_id + ), + **topk_kwargs, + ) + x_in = hidden_states + topk_ids = topk_output.topk_ids.to(torch.int32) + topk_weights = topk_output.topk_weights.to(torch.float32) + if forward_batch is not None and forward_batch.num_token_non_padded is not None: + _fill_padded_rows( + topk_ids, + forward_batch.num_token_non_padded, + experts, + ) + else: + rank_sync = envs.SGLANG_AITER_MEGA_RANK_SYNC.get() + x_in = ( + hidden_states.new_ones((1, hidden_size)) + if rank_sync + else hidden_states.new_zeros((1, hidden_size)) + ) + if rank_sync: + ep_rank, ep_world = _ep_rank_world() + expert_base = ep_rank * (experts // ep_world) + topk_ids = ( + expert_base + + torch.arange(topk, device=hidden_states.device, dtype=torch.int32) + ).unsqueeze(0) + topk_weights = hidden_states.new_full( + (1, topk), 1.0 / topk, dtype=torch.float32 + ) + else: + topk_ids = torch.full( + (1, topk), + experts, + device=hidden_states.device, + dtype=torch.int32, + ) + topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32) + + selected_mtpr = _mtpr() + x_in = x_in.contiguous() + topk_ids = topk_ids.contiguous() + topk_weights = topk_weights.contiguous() + assert x_in.shape[0] <= selected_mtpr, ( + f"Aiter MegaMoE local tokens {x_in.shape[0]} exceed MTPR {selected_mtpr}" + ) + mega = _get_or_build_mega_moe( + moe.experts, + model_dim=hidden_size, + inter_dim=moe.config.moe_intermediate_size, + experts=experts, + topk=topk, + quant=envs.SGLANG_AMD_FLYDSL_MEGA_QUANT.get() or "a8w4", + swiglu_limit=float(getattr(moe.config, "swiglu_limit", 0.0) or 0.0), + mtpr=selected_mtpr, + ) + _swap_layer_weights(mega, moe.experts) + + output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens] + + from sglang.srt.models.deepseek_common.utils import _use_aiter + + if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter): + output.mul_(moe.routed_scaling_factor) + return output diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 3175d230da..67b69edd16 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -99,6 +99,7 @@ from sglang.srt.eplb.expert_location_dispatch import ( ) from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe import ( + get_moe_a2a_backend, get_moe_runner_backend, is_moe_input_scattered_across_dp_ranks, ) @@ -2239,6 +2240,7 @@ def _post_process_topk_ids( capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids) recorder_topk_ids = None _fold_pad_into_append = False + recorder_was_fused = False if _is_cuda: # LP path: solve LP outside torch.compile (the solver contains an # EP all-reduce that can't run inside compiled regions). @@ -2302,10 +2304,34 @@ def _post_process_topk_ids( ) if not _fold_pad_into_append: _mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0) + if ( + _is_hip + and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get() + and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get() + and envs.SGLANG_AITER_MEGA_RANK_SYNC.get() + and layer_id is not None + ): + from sglang.srt.eplb.eplb_map_record_fused import ( + eplb_map_and_record_fused, + ) + + if get_moe_a2a_backend().is_megamoe(): + recorder = get_global_expert_distribution_recorder() + load_buffer = recorder.get_current_pass_count_buffer(layer_id) + if load_buffer is not None: + fused_topk_ids = eplb_map_and_record_fused( + topk_ids, + expert_location_dispatch_info, + load_buffer, + num_token_non_padded, + ) + if fused_topk_ids is not None: + topk_ids = fused_topk_ids + recorder_was_fused = True # The logical->physical remap is only meaningful when a real # expert-location mapping exists. With a trivial placement and EPLB off # the map is identity so the remap can be skipped safely. - if _eplb_remap_enabled(): + if not recorder_was_fused and _eplb_remap_enabled(): topk_ids = topk_ids_logical_to_physical( topk_ids, expert_location_dispatch_info ) @@ -2314,7 +2340,7 @@ def _post_process_topk_ids( # That final pass re-zeros after any shared-expert append/remap, so a # second zeroing here would be redundant (zeroing is idempotent). - if recorder_topk_ids is None: + if recorder_topk_ids is None and not recorder_was_fused: recorder_topk_ids = topk_ids _aiter_append = num_fused_shared_experts > 0 and _use_aiter @@ -2712,9 +2738,10 @@ def select_experts( padded_rows_masked=padded_rows_masked, ) - get_global_expert_distribution_recorder().on_select_experts( - topk_ids=recorder_topk_ids - ) + if recorder_topk_ids is not None: + get_global_expert_distribution_recorder().on_select_experts( + topk_ids=recorder_topk_ids + ) if packed_topk is not None: return StandardTopKOutputPacked( diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 242b87994c..25b39ae95e 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -1689,186 +1689,206 @@ class Fp8MoEMethod(FusedMoEMethodBase): logger.warning_once("Dequantized FP4 MoE expert weights to FP8.") def process_weights_after_loading_block_quant(self, layer: Module) -> None: - # AMD FP4 experts: use aiter's native MXFP4 MoE path. - # Skipped when dequant_fp4_to_fp8 is requested: this branch returns - # unconditionally, so without the extra check SGLANG_DSV4_FP4_DEQUANT=1 - # is silently a no-op for routed experts on every AITER deployment. - if _use_aiter and self.is_fp4_expert and not self.dequant_fp4_to_fp8: - gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get() - fp4_weight_dtype = _require_fp4_dtype() - - # DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports - # tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro - # at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%. - # shuffle_scale also supports non-256 shapes since aiter PR#4130. - fp4_k_align = 128 - E, w13_N, w13_K_packed = layer.w13_weight.shape - _, w2_N, w2_K_packed = layer.w2_weight.shape - inter_per_part = w13_N // 2 - padded_inter = ( - (inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align - ) - # Record the padding so fused_moe is told the real intermediate size - # (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes - # 128, SGLang previously defaulted to 0 -> computed the padded region). - layer.intermediate_pad = padded_inter - inter_per_part - layer.hidden_pad = 0 - if padded_inter != inter_per_part: - pad_amount = padded_inter - inter_per_part - fp4_block_k = 32 - - # Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed) - old_w13 = layer.w13_weight.data - new_w13 = torch.zeros( - E, - 2 * padded_inter, - w13_K_packed, - dtype=old_w13.dtype, - device=old_w13.device, - ) - new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :] - new_w13[:, padded_inter : padded_inter + inter_per_part, :] = old_w13[ - :, inter_per_part:, : - ] - layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False) - - # Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed) - old_w2 = layer.w2_weight.data - new_w2 = torch.zeros( - E, - w2_N, - padded_inter // 2, - dtype=old_w2.dtype, - device=old_w2.device, - ) - new_w2[:, :, :w2_K_packed] = old_w2 - layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False) - - # Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k) - old_s13 = layer.w13_weight_scale_inv.data - _, _, s13_K = old_s13.shape - new_s13 = torch.zeros( - E, - 2 * padded_inter, - s13_K, - dtype=old_s13.dtype, - device=old_s13.device, - ) - new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :] - new_s13[:, padded_inter : padded_inter + inter_per_part, :] = old_s13[ - :, inter_per_part:, : - ] - layer.w13_weight_scale_inv = torch.nn.Parameter( - new_s13, requires_grad=False + if _use_aiter and self.is_fp4_expert: + # aiter MegaMoEv2 builds from the packed FP4 layout, so it has to + # claim the experts before the native-MXFP4 and dequant arms. + if ( + get_moe_a2a_backend().is_megamoe() + and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get() + ): + if self.dequant_fp4_to_fp8: + raise ValueError( + "SGLANG_DSV4_FP4_DEQUANT strips the packed FP4 layout that " + "aiter MegaMoEv2 builds from; unset it, or pick another " + "--moe-a2a-backend." + ) + from sglang.srt.layers.moe.mega_moe import ( + build_mega_moe_experts_weights, ) - # Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k) - old_s2 = layer.w2_weight_scale_inv.data - new_s2 = torch.zeros( - E, - w2_N, - padded_inter // fp4_block_k, - dtype=old_s2.dtype, - device=old_s2.device, + fp4_weight_dtype = _require_fp4_dtype() + layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype) + layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype) + build_mega_moe_experts_weights(layer) + return + + # AMD FP4 experts: use aiter's native MXFP4 MoE path. + elif not self.dequant_fp4_to_fp8: + gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get() + fp4_weight_dtype = _require_fp4_dtype() + + # DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports + # tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro + # at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%. + # shuffle_scale also supports non-256 shapes since aiter PR#4130. + fp4_k_align = 128 + E, w13_N, w13_K_packed = layer.w13_weight.shape + _, w2_N, w2_K_packed = layer.w2_weight.shape + inter_per_part = w13_N // 2 + padded_inter = ( + (inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align ) - new_s2[:, :, : old_s2.shape[2]] = old_s2 - layer.w2_weight_scale_inv = torch.nn.Parameter( - new_s2, requires_grad=False - ) + # Record the padding so fused_moe is told the real intermediate size + # (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes + # 128, SGLang previously defaulted to 0 -> computed the padded region). + layer.intermediate_pad = padded_inter - inter_per_part + layer.hidden_pad = 0 + if padded_inter != inter_per_part: + pad_amount = padded_inter - inter_per_part + fp4_block_k = 32 + + # Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed) + old_w13 = layer.w13_weight.data + new_w13 = torch.zeros( + E, + 2 * padded_inter, + w13_K_packed, + dtype=old_w13.dtype, + device=old_w13.device, + ) + new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :] + new_w13[:, padded_inter : padded_inter + inter_per_part, :] = ( + old_w13[:, inter_per_part:, :] + ) + layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False) + + # Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed) + old_w2 = layer.w2_weight.data + new_w2 = torch.zeros( + E, + w2_N, + padded_inter // 2, + dtype=old_w2.dtype, + device=old_w2.device, + ) + new_w2[:, :, :w2_K_packed] = old_w2 + layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False) + + # Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k) + old_s13 = layer.w13_weight_scale_inv.data + _, _, s13_K = old_s13.shape + new_s13 = torch.zeros( + E, + 2 * padded_inter, + s13_K, + dtype=old_s13.dtype, + device=old_s13.device, + ) + new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :] + new_s13[:, padded_inter : padded_inter + inter_per_part, :] = ( + old_s13[:, inter_per_part:, :] + ) + layer.w13_weight_scale_inv = torch.nn.Parameter( + new_s13, requires_grad=False + ) + + # Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k) + old_s2 = layer.w2_weight_scale_inv.data + new_s2 = torch.zeros( + E, + w2_N, + padded_inter // fp4_block_k, + dtype=old_s2.dtype, + device=old_s2.device, + ) + new_s2[:, :, : old_s2.shape[2]] = old_s2 + layer.w2_weight_scale_inv = torch.nn.Parameter( + new_s2, requires_grad=False + ) + + for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"): + scale = getattr(layer, scale_name) + num_experts, num_rows, _ = scale.shape + is_w13_scale = scale_name == "w13_weight_scale_inv" + if _is_gfx1250_supported: + scale.data = moe_shuffle_scale( + scale.contiguous(), + experts_cnt=num_experts, + is_guinterleave=gu_intv, + gate_up=is_w13_scale, + ) + else: + scale_2d = scale.reshape(-1, scale.shape[-1]) + scale.data = shuffle_scale( + scale_2d, num_experts, gu_intv, is_w13_scale + ) + + layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype) + layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype) - for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"): - scale = getattr(layer, scale_name) - num_experts, num_rows, _ = scale.shape - is_w13_scale = scale_name == "w13_weight_scale_inv" if _is_gfx1250_supported: - scale.data = moe_shuffle_scale( - scale.contiguous(), - experts_cnt=num_experts, - is_guinterleave=gu_intv, - gate_up=is_w13_scale, - ) - else: - scale_2d = scale.reshape(-1, scale.shape[-1]) - scale.data = shuffle_scale( - scale_2d, num_experts, gu_intv, is_w13_scale - ) - - layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype) - layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype) - - if _is_gfx1250_supported: - is_shuffled = True - layer.w13_weight.data = moe_shuffle_weight( - layer.w13_weight, - is_guinterleave=gu_intv, - gate_up=True, - ) - layer.w2_weight.data = moe_shuffle_weight( - layer.w2_weight, - is_guinterleave=gu_intv, - gate_up=False, - ) - else: - is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4 - if is_shuffled: - shuffle_gu_intv = gu_intv and not _use_aiter_a8w4 - layer.w13_weight.data = shuffle_weight( + is_shuffled = True + layer.w13_weight.data = moe_shuffle_weight( layer.w13_weight, - is_guinterleave=shuffle_gu_intv, + is_guinterleave=gu_intv, gate_up=True, ) - layer.w2_weight.data = shuffle_weight( + layer.w2_weight.data = moe_shuffle_weight( layer.w2_weight, - is_guinterleave=shuffle_gu_intv, + is_guinterleave=gu_intv, gate_up=False, ) - layer.w13_weight.is_shuffled = is_shuffled - layer.w2_weight.is_shuffled = is_shuffled - return + else: + is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4 + if is_shuffled: + shuffle_gu_intv = gu_intv and not _use_aiter_a8w4 + layer.w13_weight.data = shuffle_weight( + layer.w13_weight, + is_guinterleave=shuffle_gu_intv, + gate_up=True, + ) + layer.w2_weight.data = shuffle_weight( + layer.w2_weight, + is_guinterleave=shuffle_gu_intv, + gate_up=False, + ) + layer.w13_weight.is_shuffled = is_shuffled + layer.w2_weight.is_shuffled = is_shuffled + return - # ROCm AITER: bypass the native FP4 early return when DSV4 dequant is - # requested, then use the standard block-FP8 MoE path. - if self.is_fp4_expert and self.dequant_fp4_to_fp8 and _use_aiter: - self._dequantize_aiter_fp4_experts(layer) - self.weight_block_size = [128, 128] + # DSV4 dequant requested: convert to block-FP8 and use the standard + # block-FP8 MoE path. + else: + self._dequantize_aiter_fp4_experts(layer) + self.weight_block_size = [128, 128] - # gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn. - if _is_fp8_fnuz: - w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( - weight=layer.w13_weight, - weight_scale=layer.w13_weight_scale_inv, - input_scale=None, - ) - w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( - weight=layer.w2_weight, - weight_scale=layer.w2_weight_scale_inv, - input_scale=None, - ) - layer.w13_weight = Parameter(w13_weight, requires_grad=False) - layer.w13_weight_scale_inv = Parameter( - w13_weight_scale, requires_grad=False - ) - layer.w2_weight = Parameter(w2_weight, requires_grad=False) - layer.w2_weight_scale_inv = Parameter( - w2_weight_scale, requires_grad=False - ) - layer.w13_input_scale = None - layer.w2_input_scale = None + # gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn. + if _is_fp8_fnuz: + w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=layer.w13_weight, + weight_scale=layer.w13_weight_scale_inv, + input_scale=None, + ) + w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=layer.w2_weight, + weight_scale=layer.w2_weight_scale_inv, + input_scale=None, + ) + layer.w13_weight = Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale_inv = Parameter( + w13_weight_scale, requires_grad=False + ) + layer.w2_weight = Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale_inv = Parameter( + w2_weight_scale, requires_grad=False + ) + layer.w13_input_scale = None + layer.w2_input_scale = None - # Only aiter-shuffle when the MoE runner is aiter; the triton runner - # consumes un-shuffled weights (shuffling the wrong runner corrupts output). - runner_is_aiter = ( - getattr(self, "runner", None) is not None - and self.runner.runner_backend.is_aiter() - ) - if _use_aiter and runner_is_aiter: - layer.w13_weight.data = shuffle_weight( - layer.w13_weight.contiguous(), (16, 16) + # Only aiter-shuffle when the MoE runner is aiter; the triton runner + # consumes un-shuffled weights (shuffling the wrong runner corrupts output). + runner_is_aiter = ( + getattr(self, "runner", None) is not None + and self.runner.runner_backend.is_aiter() ) - layer.w2_weight.data = shuffle_weight( - layer.w2_weight.contiguous(), (16, 16) - ) - return + if _use_aiter and runner_is_aiter: + layer.w13_weight.data = shuffle_weight( + layer.w13_weight.contiguous(), (16, 16) + ) + layer.w2_weight.data = shuffle_weight( + layer.w2_weight.contiguous(), (16, 16) + ) + return if self.convert_mxfp8_to_block: # Only aiter-shuffle when the MoE runner is aiter; the triton runner diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index a115ce89f6..33a0453a19 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -17,6 +17,7 @@ from sglang.srt.utils.common import ( Range, ceil_align, flatten_arrays_to_pinned_cpu, + is_hip, is_pin_memory_available, ) from sglang.srt.utils.weight_versions import ( @@ -163,6 +164,8 @@ _MM_HASH_MASK = (1 << 64) - 1 logger = logging.getLogger(__name__) +_is_hip = is_hip() + ReturnHiddenStatesMode = Union[bool, Literal["last"]] SamplingLogprobsMode = Literal["selected", "support"] @@ -2508,6 +2511,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # For DP attention global_num_tokens: Optional[List[int]] = None global_num_tokens_for_logprob: Optional[List[int]] = None + # Full DP token vector retained for Aiter MegaMoE even when the normal MLP + # TP gather path stores only this rank's token count. global_spec_verify_tier_num_tokens: Optional[List[int]] = None # Member rows riding one forward; None whenever reqs and rows are 1:1. @@ -3839,6 +3844,27 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): extend_num_tokens=self.extend_num_tokens, ) + def _swa_eviction_trigger(self, sliding_window_size: int, eviction_interval: int): + """Pick this forward's per-request SWA eviction trigger. + + Default: evict only once >= eviction_interval tokens have slid out of + the window, amortizing eviction work while keeping each request's + overshoot within the interval the pool budget reserves. Gating on + accumulated tokens rather than an iteration-counter phase cannot + starve, because seqlen progress is monotonic per KV handle. + + Aiter MegaMoE DSV4 keeps the older forward-interval cadence instead: + per-request token gating synchronizes SWA pressure across DP ranks and + triggers a retraction / re-prefill storm at high concurrency. + """ + if _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get(): + due_this_forward = (self.forward_iter or 0) % eviction_interval == 0 + return lambda req: due_this_forward + return lambda req: ( + req.seqlen - 1 - sliding_window_size + >= req.kv.swa_evicted_seqlen + eviction_interval + ) + def maybe_evict_swa(self): if self.tree_cache.supports_swa(): sliding_window_size = self.tree_cache.sliding_window_size @@ -3849,22 +3875,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): ) eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get()) + swa_evict_due = self._swa_eviction_trigger( + sliding_window_size, eviction_interval + ) self.token_to_kv_pool_allocator.free_group_begin() for idx, req in enumerate(self.reqs): if self.forward_mode.is_decode(): - # We set evict_swa condition here with two reasons: - # 1. In overlap scheduler, we cannot evict swa when req.decode_batch_idx == 0 since the prev extend batch is still running. - # 2. Evict only once >= eviction_interval tokens have slid - # out of the window, amortizing eviction work while keeping - # each request's overshoot within the interval the pool - # budget reserves. Gating on accumulated tokens (rather - # than an iteration-counter phase) cannot starve because - # seqlen progress is monotonic per KV handle. + # In overlap scheduler, we cannot evict swa when + # req.decode_batch_idx == 0 since the prev extend batch is + # still running. `_swa_eviction_trigger` owns the rest. if ( req.decode_batch_idx >= 1 and req.kv.holds_kv - and req.seqlen - 1 - sliding_window_size - >= req.kv.swa_evicted_seqlen + eviction_interval + and swa_evict_due(req) ): self._evict_swa(req, req.seqlen - 1) diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index ffa1f3c1eb..bbc4d132e1 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -226,7 +226,6 @@ def _update_gather_batch( require_mlp_tp_gather: bool, skip_global_metadata=False, ): - # TODO: handle the case when moe_dense_tp_size != 1 if not require_mlp_tp_gather: batch.global_num_tokens = [mlp_sync_info.num_tokens] batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob] @@ -493,7 +492,6 @@ def prepare_mlp_sync_batch_raw( mlp_sync_info.tp0_info_cpu[:, 4:6], ) ) - # Decide whether to emit idle batch if skip_all_gather: # Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index b47c4af0ca..f917530862 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -85,6 +85,7 @@ _skip_attn_backend_init_warned = False _is_npu = is_npu() _is_cpu = is_cpu() +_is_hip = is_hip() def _build_forward_token_modalities( @@ -155,6 +156,21 @@ def _maybe_build_forward_token_modalities( ) +def _mega_moe_materializes_idle_rank(batch: ForwardBatch) -> bool: + """Whether aiter MegaMoE needs this idle rank to carry a token anyway. + + MegaMoE's dispatch is rank-synchronous: when the batch is globally a + prefill, a rank with nothing to do still has to enter the collective, so it + runs a fabricated one-token extend instead of sitting the forward out. + """ + return bool( + _is_hip + and envs.SGLANG_AITER_MEGA_RANK_SYNC.get() + and batch.is_extend_in_batch + and batch.forward_mode.is_idle() + ) + + def _elastic_should_preserve_local_token_counts( *, model_runner: ModelRunner, @@ -1540,6 +1556,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self._original_batch_size = self.batch_size global_num_tokens = list(self.global_num_tokens_cpu) + # MegaMoEv2 dispatch is rank-synchronous, so an idle rank still carries the + # one row _run_mega_routed fabricates for it. HIP-only, False elsewhere. + mega_moe_idle_materialize = _mega_moe_materializes_idle_rank(self) + if mega_moe_idle_materialize: + global_num_tokens = [1] * len(global_num_tokens) sync_group_size = len(global_num_tokens) attn_tp_size = get_parallel().attn_tp_size @@ -1582,6 +1603,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # Preserve each rank's forward mode on coordinated steps. dp_padding_mode = DpPaddingMode.SUM_LEN self.dp_padding_mode = dp_padding_mode + # Read once here, where dp_padding_mode is final: the idle-row branch + # below runs after another arm may have rewritten self.forward_mode. + materializes_dummy_extend = mega_moe_idle_materialize or ( + self.is_extend_in_batch and dp_padding_mode.is_max_len() + ) if dp_padding_mode.is_max_len(): # when DP gather mode is all gather, we will use @@ -1634,7 +1660,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): self.forward_mode = ForwardMode.TARGET_VERIFY # Invert the spec_scale_global_num_tokens scaling. bs = self.batch_size = num_tokens // self.spec_info.num_tokens_per_req - elif self.is_extend_in_batch and dp_padding_mode.is_max_len(): + elif materializes_dummy_extend: self._original_forward_mode = self.forward_mode self.forward_mode = ForwardMode.EXTEND # Fabricate a single dummy request covering num_tokens for an diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 387a9a3a82..ad9a21bc6d 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -54,6 +54,7 @@ from sglang.srt.eplb.expert_distribution import ( ExpertDistributionRecorder, get_global_expert_distribution_recorder, set_global_expert_distribution_recorder, + should_advance_eplb_counter, ) from sglang.srt.eplb.expert_location import ( ExpertLocationMetadata, @@ -1767,7 +1768,8 @@ class ModelRunner: no_copy_to_cpu=no_copy_to_cpu, ) - if self.eplb_manager is not None: + # should_advance_eplb_counter is always True on non-hip platform + if self.eplb_manager is not None and should_advance_eplb_counter(forward_batch): self.eplb_manager.on_forward_pass_end() if dumper.may_enable: diff --git a/test/registered/unit/eplb/test_megamoe_prefill_recorder.py b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py new file mode 100644 index 0000000000..40207ec671 --- /dev/null +++ b/test/registered/unit/eplb/test_megamoe_prefill_recorder.py @@ -0,0 +1,183 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.eplb.eplb_manager import EPLBManager +from sglang.srt.eplb.eplb_map_record_fused import eplb_map_and_record_fused +from sglang.srt.eplb.expert_distribution import ( + _ExpertDistributionRecorderReal, + _SelectExpertsSinglePassGatherer, + should_advance_eplb_counter, + should_record_forward_pass, +) +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.runtime_context import override_platform +from sglang.srt.utils import Withable +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +def _batch(mode: ForwardMode, *, extend_tokens: int = 0, any_prefill: bool = False): + return SimpleNamespace( + forward_mode=mode, + extend_num_tokens=extend_tokens, + is_extend_in_batch=any_prefill, + ) + + +class TestMegaMoEPrefillRecorder(unittest.TestCase): + def setUp(self): + self.env_patches = ( + patch( + "sglang.srt.layers.moe.get_moe_a2a_backend", + return_value=SimpleNamespace(is_megamoe=lambda: True), + ), + patch( + "sglang.srt.eplb.expert_distribution.envs." + "SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get", + return_value=True, + ), + patch( + "sglang.srt.eplb.expert_distribution.envs." + "SGLANG_AITER_MEGA_RANK_SYNC.get", + return_value=True, + ), + ) + for env_patch in self.env_patches: + env_patch.start() + self.addCleanup(env_patch.stop) + capture_patch = patch( + "sglang.srt.eplb.expert_distribution._is_model_capture_mode", + return_value=False, + ) + capture_patch.start() + self.addCleanup(capture_patch.stop) + # The prefill-only arm is ROCm-only, so say so rather than inheriting + # whichever platform the test host happens to be. + platform_override = override_platform(is_hip=True) + platform_override.install() + self.addCleanup(platform_override.restore) + + def test_pass_gate_records_only_real_prefill(self): + cases = [ + (_batch(ForwardMode.EXTEND, extend_tokens=8192), True), + (_batch(ForwardMode.SPLIT_PREFILL, extend_tokens=8192), True), + (_batch(ForwardMode.EXTEND, extend_tokens=0), False), + (_batch(ForwardMode.DECODE), False), + (_batch(ForwardMode.MIXED, extend_tokens=8192), False), + (_batch(ForwardMode.IDLE), False), + (_batch(ForwardMode.TARGET_VERIFY, extend_tokens=8), False), + ] + for batch, expected in cases: + with self.subTest(mode=batch.forward_mode, extend=batch.extend_num_tokens): + self.assertEqual( + should_record_forward_pass(batch), + expected, + ) + + def test_capture_warmup_is_not_recorded(self): + batch = _batch(ForwardMode.EXTEND, extend_tokens=8192) + with patch( + "sglang.srt.eplb.expert_distribution._is_model_capture_mode", + return_value=True, + ): + self.assertFalse(should_record_forward_pass(batch)) + + def test_gate_off_preserves_legacy_decode_recording(self): + with patch( + "sglang.srt.eplb.expert_distribution.envs." + "SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get", + return_value=False, + ): + self.assertTrue(should_record_forward_pass(_batch(ForwardMode.DECODE))) + + def test_recorder_skips_decode_and_commits_prefill(self): + recorder = self._make_recorder() + recorder._on_forward_pass_start(_batch(ForwardMode.DECODE)) + recorder._on_hook("on_select_experts", topk_ids=torch.tensor([[0]])) + recorder._on_forward_pass_end(1, {}) + recorder._single_pass_gatherers["primary"].on_select_experts.assert_not_called() + + recorder._on_forward_pass_start(_batch(ForwardMode.EXTEND, extend_tokens=8192)) + recorder._on_forward_pass_end(2, {}) + recorder._accumulator.append.assert_called_once() + + def test_eplb_counter_advances_only_on_global_prefill(self): + manager = EPLBManager.__new__(EPLBManager) + advances = [] + + def generator(): + while True: + advances.append(1) + yield + + manager._main_generator = generator() + + decode = _batch(ForwardMode.DECODE, any_prefill=False) + self.assertFalse(should_advance_eplb_counter(decode)) + self.assertEqual(advances, []) + + idle_with_global_prefill = _batch(ForwardMode.IDLE, any_prefill=True) + self.assertTrue(should_advance_eplb_counter(idle_with_global_prefill)) + manager.on_forward_pass_end() + self.assertEqual(advances, [1]) + + def test_prefill_only_arm_is_inert_off_rocm(self): + decode = _batch(ForwardMode.DECODE, any_prefill=False) + with override_platform(is_hip=False): + self.assertTrue(should_advance_eplb_counter(decode)) + self.assertTrue(should_record_forward_pass(decode)) + + def test_select_experts_filters_all_out_of_range_ids(self): + gatherer = _SelectExpertsSinglePassGatherer.__new__( + _SelectExpertsSinglePassGatherer + ) + gatherer._expert_location_metadata = SimpleNamespace(num_physical_experts=4) + gatherer._data = torch.zeros((1, 4), dtype=torch.int32) + gatherer.on_select_experts( + 0, torch.tensor([[-1, 0, 3, 4, 9]], dtype=torch.int32) + ) + self.assertTrue(torch.equal(gatherer._data, torch.tensor([[1, 0, 0, 1]]))) + + @unittest.skipUnless(torch.cuda.is_available(), "requires CUDA/HIP") + def test_fused_map_record_filters_padding(self): + topk_ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int32, device="cuda") + dispatch = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device="cuda") + info = SimpleNamespace( + ep_dispatch_algorithm="static", + partial_logical_to_rank_dispatch_physical_map=dispatch, + ) + load = torch.zeros(4, dtype=torch.int32, device="cuda") + valid_tokens = torch.tensor([1], dtype=torch.int32, device="cuda") + output = eplb_map_and_record_fused(topk_ids, info, load, valid_tokens) + torch.cuda.synchronize() + self.assertTrue( + torch.equal(output.cpu(), torch.tensor([[2, 0], [3, 1]], dtype=torch.int32)) + ) + self.assertTrue( + torch.equal(load.cpu(), torch.tensor([1, 0, 1, 0], dtype=torch.int32)) + ) + + def _make_recorder(self): + recorder = _ExpertDistributionRecorderReal.__new__( + _ExpertDistributionRecorderReal + ) + gatherer = Mock() + gatherer.collect.return_value = {"global_physical_count": torch.zeros(1)} + accumulator = Mock() + accumulator.get_single_pass_gatherer_key.return_value = "primary" + recorder._recording = True + recorder._disable_all = False + recorder._record_current_pass = False + recorder._single_pass_gatherers = {"primary": gatherer} + recorder._accumulator = accumulator + recorder._current_debug_name = Withable() + recorder._current_layer_idx = SimpleNamespace(value=0) + return recorder + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_aiter_megamoe_rank_sync.py b/test/registered/unit/layers/moe/test_aiter_megamoe_rank_sync.py new file mode 100644 index 0000000000..7db7a01c6a --- /dev/null +++ b/test/registered/unit/layers/moe/test_aiter_megamoe_rank_sync.py @@ -0,0 +1,59 @@ +import sys +from types import SimpleNamespace + +import pytest +import torch + +from sglang.srt.layers.moe.mega_moe_flydsl import _should_overlap_shared_and_routed +from sglang.srt.managers.scheduler_components.dp_attn import ( + ForwardMode, + _update_gather_batch, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + + +def test_rank_sync_forces_overlap_off(monkeypatch): + monkeypatch.setenv("SGLANG_AITER_MEGA_RANK_SYNC", "1") + monkeypatch.setattr( + "sglang.srt.layers.moe.mega_moe_flydsl.get_is_capture_mode", + lambda: True, + ) + moe = SimpleNamespace(alt_stream=object(), num_fused_shared_experts=0) + assert not _should_overlap_shared_and_routed(moe, 32) + + monkeypatch.setenv("SGLANG_AITER_MEGA_RANK_SYNC", "0") + assert _should_overlap_shared_and_routed(moe, 32) + + +def test_mlp_sync_retains_full_vector_without_tp_gather(): + batch = SimpleNamespace() + sync = SimpleNamespace( + global_num_tokens=[32, 31, 7659, 256], + global_num_tokens_for_logprob=[32, 31, 1, 1], + num_tokens=32, + num_tokens_for_logprob=32, + is_extend_in_batch=True, + tbo_split_seq_index=None, + global_forward_mode=None, + can_run_decode_cuda_graph=False, + can_run_prefill_cuda_graph=False, + prefill_cuda_graph_max_prefix_len=0, + tp0_info_cpu=torch.tensor( + [ + [32, 0, 0, 0, 0, ForwardMode.DECODE.value, 0], + [31, 0, 0, 0, 0, ForwardMode.DECODE.value, 0], + [7659, 0, 0, 1, 0, ForwardMode.EXTEND.value, 0], + [256, 0, 0, 1, 0, ForwardMode.EXTEND.value, 0], + ], + dtype=torch.int64, + ), + ) + _update_gather_batch(batch, sync, require_mlp_tp_gather=False) + assert batch.global_num_tokens == [32] + assert batch.is_extend_in_batch + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 64942ca3ff..d23076c74e 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -221,22 +221,33 @@ class TestPrepareServerArgs(CustomTestCase): args.resolve_once() def test_megamoe_requires_sm90_or_sm100(self): - with override_platform(is_cuda=True, is_sm90=False, is_sm100=False): + # is_hip is pinned as well: override_platform only replaces the facts it is + # given, so on a ROCm host these would otherwise describe a machine that is + # both CUDA and HIP, and megamoe's ROCm arm would answer instead. + with override_platform( + is_cuda=True, is_sm90=False, is_sm100=False, is_hip=False + ): args = ServerArgs(model_path="dummy", moe_a2a_backend="megamoe") with self.assertRaisesRegex(ValueError, "SM90"): args.resolve_once() - with override_platform(is_cuda=False, is_sm90=False, is_sm100=False): + with override_platform( + is_cuda=False, is_sm90=False, is_sm100=False, is_hip=False + ): args = ServerArgs(model_path="dummy", moe_a2a_backend="megamoe") with self.assertRaisesRegex(ValueError, "CUDA"): args.resolve_once() - with override_platform(is_cuda=True, is_sm90=False, is_sm100=True): + with override_platform( + is_cuda=True, is_sm90=False, is_sm100=True, is_hip=False + ): ServerArgs(model_path="dummy", moe_a2a_backend="megamoe").resolve_once() def test_megamoe_token_budget_must_cover_chunked_prefill(self): from sglang.srt.arg_groups.mega_moe_hook import validate_mega_moe_token_budget from sglang.srt.environ import envs - with override_platform(is_cuda=True, is_sm90=False, is_sm100=True): + with override_platform( + is_cuda=True, is_sm90=False, is_sm100=True, is_hip=False + ): args = ServerArgs( model_path="dummy", moe_a2a_backend="megamoe",