[AMD] Integrate Aiter MegaMoEv2 for DeepSeek-V4 (#35619)

This commit is contained in:
kk
2026-09-25 00:47:08 -07:00
committed by GitHub
parent 434c2e3adc
commit 515f5be77e
15 changed files with 1088 additions and 193 deletions
@@ -36,11 +36,15 @@ def check_mega_moe_compat(server_args: ServerArgs) -> None:
"--enable-single-batch-overlap."
)
platform = get_platform()
if not (platform.is_cuda and (platform.is_sm90 or platform.is_sm100)):
if not (
(platform.is_cuda and (platform.is_sm90 or platform.is_sm100))
or platform.is_hip
):
raise ValueError(
"--moe-a2a-backend megamoe needs a CUDA SM90 GPU (block-FP8 experts) "
"or an SM100-class GPU (MXFP4 / NVFP4 experts); it runs DeepGEMM "
"kernels over CUDA symmetric memory."
"kernels over CUDA symmetric memory. On ROCm, set "
"SGLANG_AMD_USE_FLYDSL_MEGA_MOE=1 to select aiter MegaMoEv2 instead."
)
@@ -98,6 +102,8 @@ def validate_mega_moe_token_budget(server_args: ServerArgs, model_label: str) ->
cfg = resolving_view(server_args)
if cfg.moe_a2a_backend != "megamoe":
return
if get_platform().is_hip:
return
max_tokens_per_rank = (
envs.SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK.get()
+6
View File
@@ -909,6 +909,12 @@ class Envs:
SGLANG_ROCM_K3_FUSE_KDA_INPROJ_MAX_TOKENS = EnvInt(256)
SGLANG_HACK_FLASHMLA_BACKEND = EnvStr("tilelang")
SGLANG_USE_AITER_FP8_PER_TOKEN = EnvBool(False)
SGLANG_AMD_USE_FLYDSL_MEGA_MOE = EnvBool(False)
SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR = EnvInt(8192)
SGLANG_AMD_FLYDSL_MEGA_QUANT = EnvStr("")
SGLANG_AITER_MEGA_RANK_SYNC = EnvBool(False)
SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY = EnvBool(False)
SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD = EnvBool(False)
# Above 8192 tokens of context, aiter's non-static workspace is large enough
# that mem_fraction_static is scaled by 0.85 to leave room for it. Set this to
# honor an explicitly passed --mem-fraction-static instead. Off by default:
@@ -0,0 +1,114 @@
from __future__ import annotations
from typing import Optional
import torch
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
try:
import triton
import triton.language as tl
_HAS_TRITON = True
except ImportError:
_HAS_TRITON = False
if _HAS_TRITON:
@triton.jit
def _eplb_map_record_hist_kernel(
topk_ids_ptr,
dispatch_ptr,
out_ids_ptr,
load_ptr,
num_valid_tokens_ptr,
num_logical,
num_physical,
numel,
HAS_DISPATCH: tl.constexpr,
HAS_VALID_COUNT: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BINS: tl.constexpr,
TOP_K: tl.constexpr,
):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < numel
logical_id = tl.load(topk_ids_ptr + offs, mask=mask, other=-1).to(tl.int64)
valid_logical = (logical_id >= 0) & (logical_id < num_logical)
safe_logical = tl.where(valid_logical, logical_id, 0)
if HAS_DISPATCH:
physical_id = tl.load(
dispatch_ptr + safe_logical,
mask=mask & valid_logical,
other=0,
).to(tl.int64)
else:
physical_id = logical_id
output_id = tl.where(valid_logical, physical_id, logical_id)
tl.store(out_ids_ptr + offs, output_id, mask=mask)
valid_row = mask
if HAS_VALID_COUNT:
num_valid_tokens = tl.load(num_valid_tokens_ptr).to(tl.int64)
valid_row &= (offs // TOP_K) < num_valid_tokens
in_range = (
valid_row
& valid_logical
& (physical_id >= 0)
& (physical_id < num_physical)
)
bin_idx = tl.where(in_range, physical_id, num_physical).to(tl.int32)
histogram = tl.histogram(bin_idx, NUM_BINS)
bins = tl.arange(0, NUM_BINS)
tl.atomic_add(load_ptr + bins, histogram, mask=bins < num_physical)
def eplb_map_and_record_fused(
topk_ids: torch.Tensor,
dispatch_info: Optional[ExpertLocationDispatchInfo],
load_buffer: torch.Tensor,
num_token_non_padded: Optional[torch.Tensor],
) -> Optional[torch.Tensor]:
if not _HAS_TRITON or not topk_ids.is_cuda or topk_ids.numel() == 0:
return None
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm != "static":
return None
dispatch = (
dispatch_info.partial_logical_to_rank_dispatch_physical_map
if dispatch_info is not None
else None
)
if dispatch_info is not None and dispatch is None:
return None
num_physical = int(load_buffer.numel())
num_logical = int(dispatch.numel()) if dispatch is not None else num_physical
top_k = int(topk_ids.shape[-1])
numel = topk_ids.numel()
output = torch.empty_like(topk_ids)
dispatch_ptr = dispatch if dispatch is not None else topk_ids
valid_count_ptr = (
num_token_non_padded if num_token_non_padded is not None else topk_ids
)
grid = (triton.cdiv(numel, 256),)
_eplb_map_record_hist_kernel[grid](
topk_ids.contiguous(),
dispatch_ptr,
output,
load_buffer,
valid_count_ptr,
num_logical,
num_physical,
numel,
HAS_DISPATCH=dispatch is not None,
HAS_VALID_COUNT=num_token_non_padded is not None,
BLOCK=256,
NUM_BINS=1 << num_physical.bit_length(),
TOP_K=top_k,
)
return output
__all__ = ["eplb_map_and_record_fused"]
+75 -3
View File
@@ -39,7 +39,7 @@ import torch
import torch.distributed
from sglang.srt.environ import envs
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.observability.metrics_collector import (
STAT_LOGGER_ROLE_EXPERT_DISPATCH,
ExpertDispatchCollector,
@@ -48,6 +48,7 @@ from sglang.srt.observability.metrics_collector import (
from sglang.srt.runtime_context import get_device as get_device_namespace
from sglang.srt.runtime_context import (
get_exec,
get_platform,
get_schedule,
logs_expert_balancedness_to_server_log,
reports_expert_balancedness,
@@ -59,6 +60,57 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# AMD specific function
def megamoe_prefill_only_recorder_enabled() -> bool:
# The prefill-only arm belongs to aiter MegaMoEv2, which only exists on ROCm.
# Read the backend through the resolved accessor, not off a supplied
# ServerArgs: test_supplied_instance_exposure_ratchet pins (file, field)
# pairs where a `server_args` parameter is read for a field that resolution
# fills in, and that surface may only shrink.
from sglang.srt.layers.moe import get_moe_a2a_backend
return (
get_platform().is_hip
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
and get_moe_a2a_backend().is_megamoe()
)
# non-hip platform always return True
def should_record_forward_pass(forward_batch: ForwardBatch) -> bool:
if not megamoe_prefill_only_recorder_enabled():
return True
if _is_model_capture_mode():
return False
if forward_batch.forward_mode not in (
ForwardMode.EXTEND,
ForwardMode.SPLIT_PREFILL,
):
return False
return int(forward_batch.extend_num_tokens or 0) > 0
# non-hip platform always return True
def should_advance_eplb_counter(forward_batch: ForwardBatch) -> bool:
"""Whether a finished pass counts towards the EPLB rebalance interval.
Unlike `should_record_forward_pass`, which is local to this rank, the
counter has to advance in lockstep across ranks, so it keys off the globally
synchronized `is_extend_in_batch` rather than this rank's forward mode.
"""
if not megamoe_prefill_only_recorder_enabled():
return True
return forward_batch.is_extend_in_batch
def _is_model_capture_mode() -> bool:
from sglang.srt.model_executor.runner_utils.capture_mode import is_capture_mode
return is_capture_mode
# --------------------------------------- Entrypoint -----------------------------------------
_OutputMode = Literal["file", "object"]
@@ -117,6 +169,9 @@ class ExpertDistributionRecorder(ABC):
def on_select_experts(self, topk_ids: torch.Tensor):
pass
def get_current_pass_count_buffer(self, layer_idx: int):
return None
def on_deepep_dispatch_normal(
self,
local_physical_count_of_layer: List[int],
@@ -164,6 +219,7 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
self._recording = False
self._disable_all = False
self._record_current_pass = False
self._current_forward_pass_id = Withable()
self._current_layer_idx = Withable()
self._current_debug_name = Withable()
@@ -208,13 +264,17 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
def _on_forward_pass_start(self, forward_batch: ForwardBatch):
if not self._recording:
self._record_current_pass = False
return
self._record_current_pass = should_record_forward_pass(forward_batch)
if not self._record_current_pass:
return
for gatherer_key, gatherer in self._single_pass_gatherers.items():
gatherer.reset()
gatherer.on_forward_pass_start(forward_batch)
def _on_forward_pass_end(self, forward_pass_id: int, outputs: Dict[str, Any]):
if not self._recording:
if not self._recording or not self._record_current_pass:
return
for gatherer_key, gatherer in self._single_pass_gatherers.items():
single_pass_data = gatherer.collect()
@@ -225,6 +285,14 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
def on_select_experts(self, topk_ids: torch.Tensor):
self._on_hook("on_select_experts", topk_ids=topk_ids)
def get_current_pass_count_buffer(self, layer_idx: int):
if not self._recording or not self._record_current_pass:
return None
gatherer = self._single_pass_gatherers.get(_SINGLE_PASS_GATHERER_KEY_PRIMARY)
if not isinstance(gatherer, _SelectExpertsSinglePassGatherer):
return None
return gatherer._data[layer_idx]
def on_deepep_dispatch_normal(
self,
local_physical_count_of_layer: List[int],
@@ -251,6 +319,8 @@ class _ExpertDistributionRecorderReal(ExpertDistributionRecorder):
def _on_hook(self, hook_name: str, **kwargs):
if self._disable_all:
return
if megamoe_prefill_only_recorder_enabled() and not self._record_current_pass:
return
if not (
self._recording or torch.get_device_module().is_current_stream_capturing()
):
@@ -548,7 +618,9 @@ class _SelectExpertsSinglePassGatherer(_LayerBasedGpuSinglePassGatherer):
# can optimize (e.g. fuse / compile)
def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor):
topk_ids = topk_ids.flatten()
mask = topk_ids != -1
mask = (topk_ids >= 0) & (
topk_ids < self._expert_location_metadata.num_physical_experts
)
self._data[layer_idx, :].scatter_add_(
dim=0, index=topk_ids.masked_fill(~mask, 0).long(), src=mask.int()
)
+32
View File
@@ -34,6 +34,7 @@ from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.models.deepseek_common.utils import _device_sm
from sglang.srt.runtime_context import get_exec
from sglang.srt.utils import is_hip
if TYPE_CHECKING:
from deep_gemm import SymmBuffer
@@ -43,6 +44,13 @@ if TYPE_CHECKING:
_MEGA_MOE_SYMM_BUFFER: dict = {}
_is_hip = is_hip()
def _use_amd_flydsl_mega_moe() -> bool:
# aiter MegaMoEv2 exists only on ROCm; the platform check keeps the env from
# diverting a CUDA run into a path whose kernels it does not have.
return _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
def _mega_moe_mma_type(experts=None) -> str:
@@ -152,6 +160,13 @@ def is_mega_moe_experts_ready(experts) -> bool:
def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool:
if _use_amd_flydsl_mega_moe():
from sglang.srt.layers.moe.mega_moe_flydsl import (
should_use_mega_moe as should_use_flydsl_mega_moe,
)
return should_use_flydsl_mega_moe(moe, hidden_states)
if not get_moe_a2a_backend().is_megamoe():
return False
if not is_mega_moe_experts_ready(moe.experts):
@@ -174,6 +189,15 @@ def forward_mega_moe(
forward_batch: Optional[ForwardBatch] = None,
input_ids_global: Optional[torch.Tensor] = None,
) -> torch.Tensor:
if _use_amd_flydsl_mega_moe():
from sglang.srt.layers.moe.mega_moe_flydsl import (
forward_mega_moe as forward_flydsl_mega_moe,
)
return forward_flydsl_mega_moe(
moe, hidden_states, forward_batch, input_ids_global
)
num_tokens = hidden_states.shape[0]
sbo_overlap_flag = (
@@ -432,6 +456,14 @@ def _transpose_mega_moe_sf_for_utccp(sf: torch.Tensor) -> torch.Tensor:
def build_mega_moe_experts_weights(experts) -> None:
if _use_amd_flydsl_mega_moe():
from sglang.srt.layers.moe.mega_moe_flydsl import (
build_mega_moe_experts_weights as build_flydsl_mega_moe_weights,
)
build_flydsl_mega_moe_weights(experts)
return
from deep_gemm import (
transform_sf_into_required_layout,
)
@@ -0,0 +1,316 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0.
"""Aiter MegaMoEV2 adapter for the SGLang MegaMoE hooks."""
from __future__ import annotations
from contextlib import nullcontext
from types import SimpleNamespace
from typing import TYPE_CHECKING
import torch
from sglang.kernels.ops.moe.fill_padded_rows import _fill_padded_rows
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.runtime_context import get_parallel
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.models.deepseek_v2 import DeepseekV2MoE
_MORI_SHMEM_READY = False
_MEGA_MOE_INSTANCE: dict = {}
def _import_aiter_megamoe():
try:
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
from aiter.ops.shuffle import shuffle_scale_a16w4, shuffle_weight_a16w4
except ImportError as exc:
raise ImportError(
"Aiter MegaMoE requires a package containing MegaMoEV2 and the "
f"A16W4 shuffle operators. Original import error: {exc}"
) from exc
return SimpleNamespace(
op=MegaMoEV2,
shuffle_weight=shuffle_weight_a16w4,
shuffle_scale=shuffle_scale_a16w4,
)
def _mtpr() -> int:
mtpr = int(envs.SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR.get())
if mtpr <= 0 or mtpr & (mtpr - 1):
raise ValueError(
f"SGLANG_AMD_FLYDSL_MEGA_MOE_MTPR={mtpr} must be a positive power of two"
)
return mtpr
def _ep_rank_world():
parallel = get_parallel()
return parallel.moe_ep_rank, parallel.moe_ep_size
def _ensure_mori_shmem() -> None:
global _MORI_SHMEM_READY
if _MORI_SHMEM_READY:
return
import mori.shmem
group_name = "megamoe_aiter"
cpu_group = get_parallel().moe_ep_group.cpu_group
try:
torch._C._distributed_c10d._register_process_group(group_name, cpu_group)
except Exception as exc:
if "already registered" not in str(exc):
raise
mori.shmem.shmem_torch_process_group_init(group_name)
_MORI_SHMEM_READY = True
def build_mega_moe_experts_weights(layer) -> None:
if getattr(layer, "_mega_moe_weights_built", False):
return
backend = _import_aiter_megamoe()
def scale_param(name):
scale = getattr(layer, name, None)
if scale is None:
scale = getattr(layer, name + "_inv", None)
if scale is None:
raise AttributeError(f"Aiter MegaMoE weight scale {name} is missing")
return scale
def shuffle_existing(weight, scale, *, gate_up):
experts = weight.shape[0]
shuffled_weight = backend.shuffle_weight(weight, 16, gate_up).contiguous()
shuffled_scale = backend.shuffle_scale(
scale.view(-1, scale.shape[-1]), experts, gate_up
).contiguous()
return (
shuffled_weight.view(torch.uint8).contiguous(),
shuffled_scale.view(torch.uint8).contiguous(),
)
w13_scale = scale_param("w13_weight_scale")
w2_scale = scale_param("w2_weight_scale")
layer._mega_w1, layer._mega_w1_scale = shuffle_existing(
layer.w13_weight.data, w13_scale.data, gate_up=True
)
layer._mega_w2, layer._mega_w2_scale = shuffle_existing(
layer.w2_weight.data, w2_scale.data, gate_up=False
)
experts = layer.w13_weight.shape[0]
layer.w13_weight.data = layer._mega_w1.view(experts, -1)
w13_scale.data = layer._mega_w1_scale.view(experts, -1)
layer.w2_weight.data = layer._mega_w2.view(experts, -1)
w2_scale.data = layer._mega_w2_scale.view(experts, -1)
layer._mega_moe_weights_built = True
def _get_or_build_mega_moe(
layer,
*,
model_dim: int,
inter_dim: int,
experts: int,
topk: int,
quant: str,
swiglu_limit: float,
mtpr: int,
):
_ensure_mori_shmem()
backend = _import_aiter_megamoe()
rank, world = _ep_rank_world()
key = (
rank,
world,
model_dim,
inter_dim,
experts,
topk,
quant,
swiglu_limit,
mtpr,
)
mega = _MEGA_MOE_INSTANCE.get(key)
if mega is None:
mega = backend.op(
rank=rank,
world_size=world,
model_dim=model_dim,
inter_dim=inter_dim,
experts=experts,
topk=topk,
quant=quant,
w1=layer._mega_w1,
w1_scale=layer._mega_w1_scale,
w2=layer._mega_w2,
w2_scale=layer._mega_w2_scale,
max_tok_per_rank=mtpr,
swiglu_limit=swiglu_limit,
)
_MEGA_MOE_INSTANCE[key] = mega
return mega
def _swap_layer_weights(mega, layer) -> None:
mega._s1_w1 = layer._mega_w1
mega._s1_w1_scale = layer._mega_w1_scale
mega.w2 = layer._mega_w2
mega.w2_scale = layer._mega_w2_scale
def should_use_mega_moe(moe: DeepseekV2MoE, hidden_states: torch.Tensor) -> bool:
if not get_moe_a2a_backend().is_megamoe():
return False
if not getattr(moe.experts, "_mega_moe_weights_built", False):
return False
if get_is_capture_mode():
return True
global_num_tokens = get_dp_global_num_tokens()
if global_num_tokens and not is_dsa_enable_prefill_cp():
max_tokens = max(global_num_tokens)
else:
max_tokens = hidden_states.shape[0]
return max_tokens <= _mtpr()
def forward_mega_moe(
moe: DeepseekV2MoE,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch | None = None,
input_ids_global: torch.Tensor | None = None,
) -> torch.Tensor:
num_tokens = hidden_states.shape[0]
overlap = _should_overlap_shared_and_routed(moe, num_tokens)
if overlap:
current_stream = torch.cuda.current_stream()
moe.alt_stream.wait_stream(current_stream)
shared_output = moe._forward_shared_experts(hidden_states)
stream_context = torch.cuda.stream(moe.alt_stream)
else:
shared_output = moe._forward_shared_experts(hidden_states)
stream_context = nullcontext()
with stream_context:
output = _run_mega_routed(moe, hidden_states, forward_batch, input_ids_global)
if overlap:
current_stream.wait_stream(moe.alt_stream)
if shared_output is not None:
output.add_(shared_output)
return output
def _should_overlap_shared_and_routed(
moe: DeepseekV2MoE,
num_tokens: int,
) -> bool:
if envs.SGLANG_AITER_MEGA_RANK_SYNC.get():
return False
return (
moe.alt_stream is not None
and moe.num_fused_shared_experts == 0
and num_tokens > 0
and get_is_capture_mode()
)
def _run_mega_routed(
moe: DeepseekV2MoE,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch | None,
input_ids_global: torch.Tensor | None,
) -> torch.Tensor:
num_tokens = hidden_states.shape[0]
hidden_size = moe.config.hidden_size
topk = moe.config.num_experts_per_tok + moe.num_fused_shared_experts
experts = moe.experts.num_experts
if num_tokens:
router_logits = moe.gate(hidden_states, forward_batch=forward_batch)
topk_kwargs = {"input_ids": input_ids_global} if moe.is_hash else {}
topk_output = moe.topk(
hidden_states,
router_logits,
num_token_non_padded=(
forward_batch.num_token_non_padded
if forward_batch is not None
else None
),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=moe.layer_id
),
**topk_kwargs,
)
x_in = hidden_states
topk_ids = topk_output.topk_ids.to(torch.int32)
topk_weights = topk_output.topk_weights.to(torch.float32)
if forward_batch is not None and forward_batch.num_token_non_padded is not None:
_fill_padded_rows(
topk_ids,
forward_batch.num_token_non_padded,
experts,
)
else:
rank_sync = envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
x_in = (
hidden_states.new_ones((1, hidden_size))
if rank_sync
else hidden_states.new_zeros((1, hidden_size))
)
if rank_sync:
ep_rank, ep_world = _ep_rank_world()
expert_base = ep_rank * (experts // ep_world)
topk_ids = (
expert_base
+ torch.arange(topk, device=hidden_states.device, dtype=torch.int32)
).unsqueeze(0)
topk_weights = hidden_states.new_full(
(1, topk), 1.0 / topk, dtype=torch.float32
)
else:
topk_ids = torch.full(
(1, topk),
experts,
device=hidden_states.device,
dtype=torch.int32,
)
topk_weights = hidden_states.new_zeros((1, topk), dtype=torch.float32)
selected_mtpr = _mtpr()
x_in = x_in.contiguous()
topk_ids = topk_ids.contiguous()
topk_weights = topk_weights.contiguous()
assert x_in.shape[0] <= selected_mtpr, (
f"Aiter MegaMoE local tokens {x_in.shape[0]} exceed MTPR {selected_mtpr}"
)
mega = _get_or_build_mega_moe(
moe.experts,
model_dim=hidden_size,
inter_dim=moe.config.moe_intermediate_size,
experts=experts,
topk=topk,
quant=envs.SGLANG_AMD_FLYDSL_MEGA_QUANT.get() or "a8w4",
swiglu_limit=float(getattr(moe.config, "swiglu_limit", 0.0) or 0.0),
mtpr=selected_mtpr,
)
_swap_layer_weights(mega, moe.experts)
output = mega.forward(x_in, topk_weights, topk_ids, slice_output=False)[:num_tokens]
from sglang.srt.models.deepseek_common.utils import _use_aiter
if not (moe.experts.should_fuse_routed_scaling_factor_in_topk or _use_aiter):
output.mul_(moe.routed_scaling_factor)
return output
+32 -5
View File
@@ -99,6 +99,7 @@ from sglang.srt.eplb.expert_location_dispatch import (
)
from sglang.srt.layers.dp_attention import is_allocation_symmetric
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
is_moe_input_scattered_across_dp_ranks,
)
@@ -2239,6 +2240,7 @@ def _post_process_topk_ids(
capture_routed_experts_if_allowed(topk_config, layer_id, topk_ids)
recorder_topk_ids = None
_fold_pad_into_append = False
recorder_was_fused = False
if _is_cuda:
# LP path: solve LP outside torch.compile (the solver contains an
# EP all-reduce that can't run inside compiled regions).
@@ -2302,10 +2304,34 @@ def _post_process_topk_ids(
)
if not _fold_pad_into_append:
_mask_topk_ids_padded_region(topk_ids, num_token_non_padded, fill_value=0)
if (
_is_hip
and envs.SGLANG_AITER_MEGA_EPLB_PREFILL_ONLY.get()
and envs.SGLANG_AITER_MEGA_EPLB_FUSED_MAP_RECORD.get()
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
and layer_id is not None
):
from sglang.srt.eplb.eplb_map_record_fused import (
eplb_map_and_record_fused,
)
if get_moe_a2a_backend().is_megamoe():
recorder = get_global_expert_distribution_recorder()
load_buffer = recorder.get_current_pass_count_buffer(layer_id)
if load_buffer is not None:
fused_topk_ids = eplb_map_and_record_fused(
topk_ids,
expert_location_dispatch_info,
load_buffer,
num_token_non_padded,
)
if fused_topk_ids is not None:
topk_ids = fused_topk_ids
recorder_was_fused = True
# The logical->physical remap is only meaningful when a real
# expert-location mapping exists. With a trivial placement and EPLB off
# the map is identity so the remap can be skipped safely.
if _eplb_remap_enabled():
if not recorder_was_fused and _eplb_remap_enabled():
topk_ids = topk_ids_logical_to_physical(
topk_ids, expert_location_dispatch_info
)
@@ -2314,7 +2340,7 @@ def _post_process_topk_ids(
# That final pass re-zeros after any shared-expert append/remap, so a
# second zeroing here would be redundant (zeroing is idempotent).
if recorder_topk_ids is None:
if recorder_topk_ids is None and not recorder_was_fused:
recorder_topk_ids = topk_ids
_aiter_append = num_fused_shared_experts > 0 and _use_aiter
@@ -2712,9 +2738,10 @@ def select_experts(
padded_rows_masked=padded_rows_masked,
)
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_topk_ids
)
if recorder_topk_ids is not None:
get_global_expert_distribution_recorder().on_select_experts(
topk_ids=recorder_topk_ids
)
if packed_topk is not None:
return StandardTopKOutputPacked(
+185 -165
View File
@@ -1689,186 +1689,206 @@ class Fp8MoEMethod(FusedMoEMethodBase):
logger.warning_once("Dequantized FP4 MoE expert weights to FP8.")
def process_weights_after_loading_block_quant(self, layer: Module) -> None:
# AMD FP4 experts: use aiter's native MXFP4 MoE path.
# Skipped when dequant_fp4_to_fp8 is requested: this branch returns
# unconditionally, so without the extra check SGLANG_DSV4_FP4_DEQUANT=1
# is silently a no-op for routed experts on every AITER deployment.
if _use_aiter and self.is_fp4_expert and not self.dequant_fp4_to_fp8:
gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
fp4_weight_dtype = _require_fp4_dtype()
# DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports
# tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro
# at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%.
# shuffle_scale also supports non-256 shapes since aiter PR#4130.
fp4_k_align = 128
E, w13_N, w13_K_packed = layer.w13_weight.shape
_, w2_N, w2_K_packed = layer.w2_weight.shape
inter_per_part = w13_N // 2
padded_inter = (
(inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align
)
# Record the padding so fused_moe is told the real intermediate size
# (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes
# 128, SGLang previously defaulted to 0 -> computed the padded region).
layer.intermediate_pad = padded_inter - inter_per_part
layer.hidden_pad = 0
if padded_inter != inter_per_part:
pad_amount = padded_inter - inter_per_part
fp4_block_k = 32
# Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed)
old_w13 = layer.w13_weight.data
new_w13 = torch.zeros(
E,
2 * padded_inter,
w13_K_packed,
dtype=old_w13.dtype,
device=old_w13.device,
)
new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :]
new_w13[:, padded_inter : padded_inter + inter_per_part, :] = old_w13[
:, inter_per_part:, :
]
layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False)
# Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed)
old_w2 = layer.w2_weight.data
new_w2 = torch.zeros(
E,
w2_N,
padded_inter // 2,
dtype=old_w2.dtype,
device=old_w2.device,
)
new_w2[:, :, :w2_K_packed] = old_w2
layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False)
# Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k)
old_s13 = layer.w13_weight_scale_inv.data
_, _, s13_K = old_s13.shape
new_s13 = torch.zeros(
E,
2 * padded_inter,
s13_K,
dtype=old_s13.dtype,
device=old_s13.device,
)
new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :]
new_s13[:, padded_inter : padded_inter + inter_per_part, :] = old_s13[
:, inter_per_part:, :
]
layer.w13_weight_scale_inv = torch.nn.Parameter(
new_s13, requires_grad=False
if _use_aiter and self.is_fp4_expert:
# aiter MegaMoEv2 builds from the packed FP4 layout, so it has to
# claim the experts before the native-MXFP4 and dequant arms.
if (
get_moe_a2a_backend().is_megamoe()
and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get()
):
if self.dequant_fp4_to_fp8:
raise ValueError(
"SGLANG_DSV4_FP4_DEQUANT strips the packed FP4 layout that "
"aiter MegaMoEv2 builds from; unset it, or pick another "
"--moe-a2a-backend."
)
from sglang.srt.layers.moe.mega_moe import (
build_mega_moe_experts_weights,
)
# Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k)
old_s2 = layer.w2_weight_scale_inv.data
new_s2 = torch.zeros(
E,
w2_N,
padded_inter // fp4_block_k,
dtype=old_s2.dtype,
device=old_s2.device,
fp4_weight_dtype = _require_fp4_dtype()
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
build_mega_moe_experts_weights(layer)
return
# AMD FP4 experts: use aiter's native MXFP4 MoE path.
elif not self.dequant_fp4_to_fp8:
gu_intv = envs.SGLANG_USE_AITER_MOE_GU_ITLV.get()
fp4_weight_dtype = _require_fp4_dtype()
# DeepSeek V4 MoE is implemented by the FlyDSL kernel, which supports
# tile_k=128, so we only need to pad dim to 128. This lets DeepSeek-V4-Pro
# at TP8 skip padding 384 -> 512, reducing routed-expert memory by ~25%.
# shuffle_scale also supports non-256 shapes since aiter PR#4130.
fp4_k_align = 128
E, w13_N, w13_K_packed = layer.w13_weight.shape
_, w2_N, w2_K_packed = layer.w2_weight.shape
inter_per_part = w13_N // 2
padded_inter = (
(inter_per_part + fp4_k_align - 1) // fp4_k_align * fp4_k_align
)
new_s2[:, :, : old_s2.shape[2]] = old_s2
layer.w2_weight_scale_inv = torch.nn.Parameter(
new_s2, requires_grad=False
)
# Record the padding so fused_moe is told the real intermediate size
# (aiter fused_moe needs intermediate_pad = padded - real; ATOM passes
# 128, SGLang previously defaulted to 0 -> computed the padded region).
layer.intermediate_pad = padded_inter - inter_per_part
layer.hidden_pad = 0
if padded_inter != inter_per_part:
pad_amount = padded_inter - inter_per_part
fp4_block_k = 32
# Pad w13_weight: (E, 2*inter, K_packed) → (E, 2*padded, K_packed)
old_w13 = layer.w13_weight.data
new_w13 = torch.zeros(
E,
2 * padded_inter,
w13_K_packed,
dtype=old_w13.dtype,
device=old_w13.device,
)
new_w13[:, :inter_per_part, :] = old_w13[:, :inter_per_part, :]
new_w13[:, padded_inter : padded_inter + inter_per_part, :] = (
old_w13[:, inter_per_part:, :]
)
layer.w13_weight = torch.nn.Parameter(new_w13, requires_grad=False)
# Pad w2_weight: (E, N, inter_packed) → (E, N, padded_packed)
old_w2 = layer.w2_weight.data
new_w2 = torch.zeros(
E,
w2_N,
padded_inter // 2,
dtype=old_w2.dtype,
device=old_w2.device,
)
new_w2[:, :, :w2_K_packed] = old_w2
layer.w2_weight = torch.nn.Parameter(new_w2, requires_grad=False)
# Pad w13 scale: (E, 2*inter, K/block_k) → (E, 2*padded, K/block_k)
old_s13 = layer.w13_weight_scale_inv.data
_, _, s13_K = old_s13.shape
new_s13 = torch.zeros(
E,
2 * padded_inter,
s13_K,
dtype=old_s13.dtype,
device=old_s13.device,
)
new_s13[:, :inter_per_part, :] = old_s13[:, :inter_per_part, :]
new_s13[:, padded_inter : padded_inter + inter_per_part, :] = (
old_s13[:, inter_per_part:, :]
)
layer.w13_weight_scale_inv = torch.nn.Parameter(
new_s13, requires_grad=False
)
# Pad w2 scale: (E, N, inter/block_k) → (E, N, padded/block_k)
old_s2 = layer.w2_weight_scale_inv.data
new_s2 = torch.zeros(
E,
w2_N,
padded_inter // fp4_block_k,
dtype=old_s2.dtype,
device=old_s2.device,
)
new_s2[:, :, : old_s2.shape[2]] = old_s2
layer.w2_weight_scale_inv = torch.nn.Parameter(
new_s2, requires_grad=False
)
for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"):
scale = getattr(layer, scale_name)
num_experts, num_rows, _ = scale.shape
is_w13_scale = scale_name == "w13_weight_scale_inv"
if _is_gfx1250_supported:
scale.data = moe_shuffle_scale(
scale.contiguous(),
experts_cnt=num_experts,
is_guinterleave=gu_intv,
gate_up=is_w13_scale,
)
else:
scale_2d = scale.reshape(-1, scale.shape[-1])
scale.data = shuffle_scale(
scale_2d, num_experts, gu_intv, is_w13_scale
)
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
for scale_name in ("w13_weight_scale_inv", "w2_weight_scale_inv"):
scale = getattr(layer, scale_name)
num_experts, num_rows, _ = scale.shape
is_w13_scale = scale_name == "w13_weight_scale_inv"
if _is_gfx1250_supported:
scale.data = moe_shuffle_scale(
scale.contiguous(),
experts_cnt=num_experts,
is_guinterleave=gu_intv,
gate_up=is_w13_scale,
)
else:
scale_2d = scale.reshape(-1, scale.shape[-1])
scale.data = shuffle_scale(
scale_2d, num_experts, gu_intv, is_w13_scale
)
layer.w13_weight.data = layer.w13_weight.data.view(fp4_weight_dtype)
layer.w2_weight.data = layer.w2_weight.data.view(fp4_weight_dtype)
if _is_gfx1250_supported:
is_shuffled = True
layer.w13_weight.data = moe_shuffle_weight(
layer.w13_weight,
is_guinterleave=gu_intv,
gate_up=True,
)
layer.w2_weight.data = moe_shuffle_weight(
layer.w2_weight,
is_guinterleave=gu_intv,
gate_up=False,
)
else:
is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4
if is_shuffled:
shuffle_gu_intv = gu_intv and not _use_aiter_a8w4
layer.w13_weight.data = shuffle_weight(
is_shuffled = True
layer.w13_weight.data = moe_shuffle_weight(
layer.w13_weight,
is_guinterleave=shuffle_gu_intv,
is_guinterleave=gu_intv,
gate_up=True,
)
layer.w2_weight.data = shuffle_weight(
layer.w2_weight.data = moe_shuffle_weight(
layer.w2_weight,
is_guinterleave=shuffle_gu_intv,
is_guinterleave=gu_intv,
gate_up=False,
)
layer.w13_weight.is_shuffled = is_shuffled
layer.w2_weight.is_shuffled = is_shuffled
return
else:
is_shuffled = _is_shuffle_moe_mxfp4 or _use_aiter_a8w4
if is_shuffled:
shuffle_gu_intv = gu_intv and not _use_aiter_a8w4
layer.w13_weight.data = shuffle_weight(
layer.w13_weight,
is_guinterleave=shuffle_gu_intv,
gate_up=True,
)
layer.w2_weight.data = shuffle_weight(
layer.w2_weight,
is_guinterleave=shuffle_gu_intv,
gate_up=False,
)
layer.w13_weight.is_shuffled = is_shuffled
layer.w2_weight.is_shuffled = is_shuffled
return
# ROCm AITER: bypass the native FP4 early return when DSV4 dequant is
# requested, then use the standard block-FP8 MoE path.
if self.is_fp4_expert and self.dequant_fp4_to_fp8 and _use_aiter:
self._dequantize_aiter_fp4_experts(layer)
self.weight_block_size = [128, 128]
# DSV4 dequant requested: convert to block-FP8 and use the standard
# block-FP8 MoE path.
else:
self._dequantize_aiter_fp4_experts(layer)
self.weight_block_size = [128, 128]
# gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn.
if _is_fp8_fnuz:
w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.w13_weight,
weight_scale=layer.w13_weight_scale_inv,
input_scale=None,
)
w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.w2_weight,
weight_scale=layer.w2_weight_scale_inv,
input_scale=None,
)
layer.w13_weight = Parameter(w13_weight, requires_grad=False)
layer.w13_weight_scale_inv = Parameter(
w13_weight_scale, requires_grad=False
)
layer.w2_weight = Parameter(w2_weight, requires_grad=False)
layer.w2_weight_scale_inv = Parameter(
w2_weight_scale, requires_grad=False
)
layer.w13_input_scale = None
layer.w2_input_scale = None
# gfx942/gfx950 native FP8 is e4m3fnuz, not e4m3fn.
if _is_fp8_fnuz:
w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.w13_weight,
weight_scale=layer.w13_weight_scale_inv,
input_scale=None,
)
w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.w2_weight,
weight_scale=layer.w2_weight_scale_inv,
input_scale=None,
)
layer.w13_weight = Parameter(w13_weight, requires_grad=False)
layer.w13_weight_scale_inv = Parameter(
w13_weight_scale, requires_grad=False
)
layer.w2_weight = Parameter(w2_weight, requires_grad=False)
layer.w2_weight_scale_inv = Parameter(
w2_weight_scale, requires_grad=False
)
layer.w13_input_scale = None
layer.w2_input_scale = None
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
# consumes un-shuffled weights (shuffling the wrong runner corrupts output).
runner_is_aiter = (
getattr(self, "runner", None) is not None
and self.runner.runner_backend.is_aiter()
)
if _use_aiter and runner_is_aiter:
layer.w13_weight.data = shuffle_weight(
layer.w13_weight.contiguous(), (16, 16)
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
# consumes un-shuffled weights (shuffling the wrong runner corrupts output).
runner_is_aiter = (
getattr(self, "runner", None) is not None
and self.runner.runner_backend.is_aiter()
)
layer.w2_weight.data = shuffle_weight(
layer.w2_weight.contiguous(), (16, 16)
)
return
if _use_aiter and runner_is_aiter:
layer.w13_weight.data = shuffle_weight(
layer.w13_weight.contiguous(), (16, 16)
)
layer.w2_weight.data = shuffle_weight(
layer.w2_weight.contiguous(), (16, 16)
)
return
if self.convert_mxfp8_to_block:
# Only aiter-shuffle when the MoE runner is aiter; the triton runner
+33 -10
View File
@@ -17,6 +17,7 @@ from sglang.srt.utils.common import (
Range,
ceil_align,
flatten_arrays_to_pinned_cpu,
is_hip,
is_pin_memory_available,
)
from sglang.srt.utils.weight_versions import (
@@ -163,6 +164,8 @@ _MM_HASH_MASK = (1 << 64) - 1
logger = logging.getLogger(__name__)
_is_hip = is_hip()
ReturnHiddenStatesMode = Union[bool, Literal["last"]]
SamplingLogprobsMode = Literal["selected", "support"]
@@ -2508,6 +2511,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# For DP attention
global_num_tokens: Optional[List[int]] = None
global_num_tokens_for_logprob: Optional[List[int]] = None
# Full DP token vector retained for Aiter MegaMoE even when the normal MLP
# TP gather path stores only this rank's token count.
global_spec_verify_tier_num_tokens: Optional[List[int]] = None
# Member rows riding one forward; None whenever reqs and rows are 1:1.
@@ -3839,6 +3844,27 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
extend_num_tokens=self.extend_num_tokens,
)
def _swa_eviction_trigger(self, sliding_window_size: int, eviction_interval: int):
"""Pick this forward's per-request SWA eviction trigger.
Default: evict only once >= eviction_interval tokens have slid out of
the window, amortizing eviction work while keeping each request's
overshoot within the interval the pool budget reserves. Gating on
accumulated tokens rather than an iteration-counter phase cannot
starve, because seqlen progress is monotonic per KV handle.
Aiter MegaMoE DSV4 keeps the older forward-interval cadence instead:
per-request token gating synchronizes SWA pressure across DP ranks and
triggers a retraction / re-prefill storm at high concurrency.
"""
if _is_hip and envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.get():
due_this_forward = (self.forward_iter or 0) % eviction_interval == 0
return lambda req: due_this_forward
return lambda req: (
req.seqlen - 1 - sliding_window_size
>= req.kv.swa_evicted_seqlen + eviction_interval
)
def maybe_evict_swa(self):
if self.tree_cache.supports_swa():
sliding_window_size = self.tree_cache.sliding_window_size
@@ -3849,22 +3875,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
)
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
swa_evict_due = self._swa_eviction_trigger(
sliding_window_size, eviction_interval
)
self.token_to_kv_pool_allocator.free_group_begin()
for idx, req in enumerate(self.reqs):
if self.forward_mode.is_decode():
# We set evict_swa condition here with two reasons:
# 1. In overlap scheduler, we cannot evict swa when req.decode_batch_idx == 0 since the prev extend batch is still running.
# 2. Evict only once >= eviction_interval tokens have slid
# out of the window, amortizing eviction work while keeping
# each request's overshoot within the interval the pool
# budget reserves. Gating on accumulated tokens (rather
# than an iteration-counter phase) cannot starve because
# seqlen progress is monotonic per KV handle.
# In overlap scheduler, we cannot evict swa when
# req.decode_batch_idx == 0 since the prev extend batch is
# still running. `_swa_eviction_trigger` owns the rest.
if (
req.decode_batch_idx >= 1
and req.kv.holds_kv
and req.seqlen - 1 - sliding_window_size
>= req.kv.swa_evicted_seqlen + eviction_interval
and swa_evict_due(req)
):
self._evict_swa(req, req.seqlen - 1)
@@ -226,7 +226,6 @@ def _update_gather_batch(
require_mlp_tp_gather: bool,
skip_global_metadata=False,
):
# TODO: handle the case when moe_dense_tp_size != 1
if not require_mlp_tp_gather:
batch.global_num_tokens = [mlp_sync_info.num_tokens]
batch.global_num_tokens_for_logprob = [mlp_sync_info.num_tokens_for_logprob]
@@ -493,7 +492,6 @@ def prepare_mlp_sync_batch_raw(
mlp_sync_info.tp0_info_cpu[:, 4:6],
)
)
# Decide whether to emit idle batch
if skip_all_gather:
# Skip idle batch when attn-dp=1 (and always under DWDP: ranks run independently)
@@ -85,6 +85,7 @@ _skip_attn_backend_init_warned = False
_is_npu = is_npu()
_is_cpu = is_cpu()
_is_hip = is_hip()
def _build_forward_token_modalities(
@@ -155,6 +156,21 @@ def _maybe_build_forward_token_modalities(
)
def _mega_moe_materializes_idle_rank(batch: ForwardBatch) -> bool:
"""Whether aiter MegaMoE needs this idle rank to carry a token anyway.
MegaMoE's dispatch is rank-synchronous: when the batch is globally a
prefill, a rank with nothing to do still has to enter the collective, so it
runs a fabricated one-token extend instead of sitting the forward out.
"""
return bool(
_is_hip
and envs.SGLANG_AITER_MEGA_RANK_SYNC.get()
and batch.is_extend_in_batch
and batch.forward_mode.is_idle()
)
def _elastic_should_preserve_local_token_counts(
*,
model_runner: ModelRunner,
@@ -1540,6 +1556,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self._original_batch_size = self.batch_size
global_num_tokens = list(self.global_num_tokens_cpu)
# MegaMoEv2 dispatch is rank-synchronous, so an idle rank still carries the
# one row _run_mega_routed fabricates for it. HIP-only, False elsewhere.
mega_moe_idle_materialize = _mega_moe_materializes_idle_rank(self)
if mega_moe_idle_materialize:
global_num_tokens = [1] * len(global_num_tokens)
sync_group_size = len(global_num_tokens)
attn_tp_size = get_parallel().attn_tp_size
@@ -1582,6 +1603,11 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# Preserve each rank's forward mode on coordinated steps.
dp_padding_mode = DpPaddingMode.SUM_LEN
self.dp_padding_mode = dp_padding_mode
# Read once here, where dp_padding_mode is final: the idle-row branch
# below runs after another arm may have rewritten self.forward_mode.
materializes_dummy_extend = mega_moe_idle_materialize or (
self.is_extend_in_batch and dp_padding_mode.is_max_len()
)
if dp_padding_mode.is_max_len():
# when DP gather mode is all gather, we will use
@@ -1634,7 +1660,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.forward_mode = ForwardMode.TARGET_VERIFY
# Invert the spec_scale_global_num_tokens scaling.
bs = self.batch_size = num_tokens // self.spec_info.num_tokens_per_req
elif self.is_extend_in_batch and dp_padding_mode.is_max_len():
elif materializes_dummy_extend:
self._original_forward_mode = self.forward_mode
self.forward_mode = ForwardMode.EXTEND
# Fabricate a single dummy request covering num_tokens for an
@@ -54,6 +54,7 @@ from sglang.srt.eplb.expert_distribution import (
ExpertDistributionRecorder,
get_global_expert_distribution_recorder,
set_global_expert_distribution_recorder,
should_advance_eplb_counter,
)
from sglang.srt.eplb.expert_location import (
ExpertLocationMetadata,
@@ -1767,7 +1768,8 @@ class ModelRunner:
no_copy_to_cpu=no_copy_to_cpu,
)
if self.eplb_manager is not None:
# should_advance_eplb_counter is always True on non-hip platform
if self.eplb_manager is not None and should_advance_eplb_counter(forward_batch):
self.eplb_manager.on_forward_pass_end()
if dumper.may_enable: