[AMD][MORI] Add MORI EP V2 support (#40666)
This commit is contained in:
@@ -30,7 +30,12 @@ from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
|
||||
from sglang.srt.utils.common import is_sm100_supported, parse_connector_type
|
||||
from sglang.srt.utils.common import (
|
||||
get_bool_env_var,
|
||||
is_gfx1250_supported,
|
||||
is_sm100_supported,
|
||||
parse_connector_type,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -460,6 +465,7 @@ def handle_a2a_moe(server_args: Any):
|
||||
)
|
||||
|
||||
if a2a_backend == "mori":
|
||||
is_epv2 = _resolve_mori_ep_v2(server_args)
|
||||
if cfg.deepep_mode == "auto":
|
||||
declare_resolution(
|
||||
server_args,
|
||||
@@ -467,6 +473,16 @@ def handle_a2a_moe(server_args: Any):
|
||||
deepep_mode="normal",
|
||||
)
|
||||
logger.warning("auto set deepep_mode=`normal` for MORI EP")
|
||||
elif is_epv2 and cfg.deepep_mode == "low_latency":
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_handle_a2a_moe",
|
||||
deepep_mode="normal",
|
||||
)
|
||||
logger.warning(
|
||||
"MORI EPv2 has no separate low-latency kernel; "
|
||||
"using deepep_mode=`normal` for deepep_mode=`low_latency`"
|
||||
)
|
||||
|
||||
# Check chunked prefill for mori
|
||||
# Skip validation if chunked prefill is disabled (i.e., size <= 0).
|
||||
@@ -682,6 +698,63 @@ def validate_flashinfer_a2a_token_budget(server_args: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _resolve_mori_ep_v2(server_args: Any) -> bool:
|
||||
"""Whether to use MORI EPv2.
|
||||
|
||||
Unset prefers EPv2 and falls back to EPv1 where it is unsupported; an explicit
|
||||
true keeps EPv2 regardless, and false selects EPv1.
|
||||
"""
|
||||
if not envs.SGLANG_MORI_EP_V2.get():
|
||||
return False
|
||||
|
||||
reason = _mori_epv2_unsupported_reason(server_args)
|
||||
if reason is None:
|
||||
return True
|
||||
if envs.SGLANG_MORI_EP_V2.is_set():
|
||||
logger.warning(
|
||||
"SGLANG_MORI_EP_V2=true is set explicitly, so MORI EPv2 is used even "
|
||||
"though it does not support %s.",
|
||||
reason,
|
||||
)
|
||||
return True
|
||||
|
||||
logger.warning(
|
||||
"MORI EPv2 does not support %s; falling back to MORI EPv1. "
|
||||
"Set SGLANG_MORI_EP_V2=true to force EPv2, or false to select EPv1.",
|
||||
reason,
|
||||
)
|
||||
# Worker processes inherit the env, so every reader agrees on EPv1.
|
||||
envs.SGLANG_MORI_EP_V2.set(False)
|
||||
return False
|
||||
|
||||
|
||||
def _mori_epv2_unsupported_reason(server_args: Any) -> str | None:
|
||||
# The dispatcher falls back from unsupported dispatch/combine dtypes once it
|
||||
# knows the weight dtype.
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.model_config import is_deepseek_v4
|
||||
|
||||
cfg = resolved_view(server_args)
|
||||
# gfx1250 forms its LSA team from the UALink vPOD, which can span hosts and is
|
||||
# only known once cco is up, so init_mori_epv2_op checks that EP fits in it.
|
||||
if cfg.tp_size > 8 and not is_gfx1250_supported():
|
||||
return f"ep_size={cfg.tp_size} across hosts (EPv2 is validated within one host)"
|
||||
if not envs.SGLANG_USE_AITER.get():
|
||||
return "SGLANG_USE_AITER=0 (EPv2 experts run on the aiter MoE runner)"
|
||||
if get_bool_env_var("MORI_ENABLE_SDMA", "false"):
|
||||
return "MORI_ENABLE_SDMA=1 (SDMA is EPv1-only)"
|
||||
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
|
||||
return "a model loaded through an instance connector (cannot validate it)"
|
||||
model_config = model_config_of(server_args)
|
||||
if model_config.dtype != torch.bfloat16:
|
||||
return f"dtype={model_config.dtype} (EPv2 requires bfloat16)"
|
||||
if not is_deepseek_v4(model_config.hf_config):
|
||||
architecture = (model_config.hf_config.architectures or [None])[0]
|
||||
return f"{architecture} (EPv2 is validated on DeepSeek-V4 only)"
|
||||
return None
|
||||
|
||||
|
||||
def required_mori_dispatch_tokens_per_rank(server_args: Any) -> int:
|
||||
"""Max tokens a single rank dispatches through MoRI in one forward."""
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
@@ -864,6 +864,17 @@ class Envs:
|
||||
# and relies on the RDMA retry-exceeded timeout only.
|
||||
SGLANG_MORI_TRANSFER_TIMEOUT_MS = EnvInt(0)
|
||||
SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(4096)
|
||||
# Unset: EPv2 where moe_hook finds it supported (DeepSeek-V4 only), else EPv1.
|
||||
# true forces EPv2; false forces EPv1.
|
||||
SGLANG_MORI_EP_V2 = EnvBool(True)
|
||||
SGLANG_MORI_DISPATCH_DTYPE = EnvStr("auto")
|
||||
SGLANG_MORI_COMBINE_DTYPE = EnvStr("auto")
|
||||
SGLANG_MORI_PREALLOC_MAX_RECV_TOKENS = EnvInt(0)
|
||||
# Hand AITER only ceil32(sum of per-rank DP tokens) rows of the MORI EP receive
|
||||
# buffer; unproved layouts or metadata keep the full view. Forces DP-synced buckets.
|
||||
SGLANG_MORI_RECV_BOUND = EnvBool(False)
|
||||
# Manual MoE input row limit (EPv1 and EPv2); 0 leaves it to SGLANG_MORI_RECV_BOUND.
|
||||
SGLANG_MORI_MOE_MAX_INPUT_TOKENS = EnvInt(0)
|
||||
|
||||
# ===================================================================
|
||||
# AMD, ROCm, and AITER
|
||||
|
||||
@@ -1092,9 +1092,14 @@ class _InfiniteBuffer(_Buffer):
|
||||
device = self._buffer.device
|
||||
|
||||
if self._size == curr_buffer_size:
|
||||
new_buffer = torch.zeros(
|
||||
(2 * curr_buffer_size, *self._item_shape), dtype=dtype, device=device
|
||||
)
|
||||
# Growth can happen inside a caller's inference_mode (e.g. DSpark draft);
|
||||
# an inference tensor buffer would reject later appends made outside it.
|
||||
with torch.inference_mode(False):
|
||||
new_buffer = torch.zeros(
|
||||
(2 * curr_buffer_size, *self._item_shape),
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
new_buffer[:curr_buffer_size] = self._buffer
|
||||
self._buffer = new_buffer
|
||||
|
||||
|
||||
@@ -352,6 +352,8 @@ class DeepEPMoE(FusedMoE):
|
||||
|
||||
|
||||
def get_moe_impl_class(quant_config: Optional[QuantizationConfig]):
|
||||
if get_moe_a2a_backend().is_mori_epv2():
|
||||
return FusedMoE
|
||||
# [TODO] kk, temporary solution
|
||||
if (
|
||||
get_moe_a2a_backend().is_mori()
|
||||
|
||||
@@ -23,8 +23,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
|
||||
register_pre_permute,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import MoeRunnerBackend
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, get_int_env_var, is_gfx95_supported
|
||||
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher.base import CombineInput
|
||||
@@ -87,6 +86,8 @@ class AiterRunnerInput(RunnerInput):
|
||||
# Mori-only fused_moe kwargs.
|
||||
num_local_tokens: Optional[torch.Tensor] = None
|
||||
output_dtype: Optional[torch.dtype] = None
|
||||
# Optional MORI EPv2 combine staging view for fused_moe output.
|
||||
output: Optional[torch.Tensor] = None
|
||||
|
||||
@property
|
||||
def runner_backend(self) -> MoeRunnerBackend:
|
||||
@@ -156,95 +157,11 @@ def _aiter_fused_moe_supports_no_combine() -> bool:
|
||||
return "no_combine" in inspect.signature(fused_moe).parameters
|
||||
|
||||
|
||||
_RECV_BOUND_LOGGED: set[int] = set()
|
||||
_RECV_BOUND_WARNED = False
|
||||
@functools.cache
|
||||
def _aiter_fused_moe_supports_output() -> bool:
|
||||
from aiter.fused_moe import fused_moe
|
||||
|
||||
|
||||
def _warn_recv_bound_unavailable() -> None:
|
||||
global _RECV_BOUND_WARNED
|
||||
if not _RECV_BOUND_WARNED:
|
||||
_RECV_BOUND_WARNED = True
|
||||
logger.warning(
|
||||
"SGLANG_MORI_RECV_BOUND is set but the per-rank DP token counts do "
|
||||
"not cover every mori sender, so the receive fan-in is unknown; "
|
||||
"leaving the receive buffer unbounded."
|
||||
)
|
||||
|
||||
|
||||
def _mori_decode_recv_bound(recv_rows: int, topk: int) -> int:
|
||||
"""Live rows mori's receive buffer can hold in decode, or 0 for "do not bound".
|
||||
|
||||
Worst case fan-in is every rank routing all of its tokens to this one, so
|
||||
`sum(per-rank tokens) * topk`, where topk already includes the fused shared
|
||||
expert. The per-rank counts come from the DP sync, so this is the fan-in for
|
||||
the batch actually being run rather than an upper bound over all batches.
|
||||
|
||||
That is only sound because enabling this gate also makes
|
||||
`require_mlp_tp_gather()` true for mori, which gives every rank the same
|
||||
cuda-graph bucket. The value is baked into a captured graph and has to hold
|
||||
for every later replay; with per-rank buckets a rank on a narrow tier could
|
||||
be handed rows by a peer on a wider one, and the only bound valid under that
|
||||
is the widest tier's -- 4-16x looser than the batch being run, which costs
|
||||
more in expert-GEMM tiles (M 32/64 -> 128) than the trim saves.
|
||||
|
||||
Two cases stay unbounded, because a bound below the real fan-in silently
|
||||
drops rows from the all-to-all -- wrong output rather than an error:
|
||||
|
||||
* Prefill, whose per-rank counts are uneven and not knowable here.
|
||||
* Anything that leaves the per-rank counts unpopulated, or where the EP world
|
||||
is wider than the DP world so the counts do not cover every sender.
|
||||
"""
|
||||
if not get_bool_env_var("SGLANG_MORI_RECV_BOUND", "false"):
|
||||
return 0
|
||||
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
get_is_extend_in_batch,
|
||||
)
|
||||
|
||||
if get_is_extend_in_batch():
|
||||
return 0
|
||||
|
||||
per_rank_tokens = get_dp_global_num_tokens()
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
if not per_rank_tokens or len(per_rank_tokens) < ep_size:
|
||||
# Either the DP sync did not publish counts, or they do not cover every
|
||||
# mori sender. Both mean the fan-in is unknown here.
|
||||
_warn_recv_bound_unavailable()
|
||||
return 0
|
||||
|
||||
max_tokens = sum(per_rank_tokens)
|
||||
bound = max_tokens * topk
|
||||
# Never grow the tensor, and nothing to do when there is nothing to trim.
|
||||
if not 0 < bound < recv_rows:
|
||||
return 0
|
||||
|
||||
# One INFO line the first time it engages, so an inert bound is not mistaken
|
||||
# for an active one in the results. Per-tier values go to DEBUG: capture
|
||||
# visits every tier, and at INFO on every rank that is dozens of lines.
|
||||
if get_parallel().tp_rank == 0 and bound not in _RECV_BOUND_LOGGED:
|
||||
first = not _RECV_BOUND_LOGGED
|
||||
_RECV_BOUND_LOGGED.add(bound)
|
||||
if first:
|
||||
logger.info(
|
||||
"mori recv bound active: %d rows -> %d for this tier "
|
||||
"(dp_tokens=%d ep=%d topk=%d); per-tier values at DEBUG",
|
||||
recv_rows,
|
||||
bound,
|
||||
max_tokens,
|
||||
get_parallel().moe_ep_size,
|
||||
topk,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"mori recv bound: %d rows -> %d (dp_tokens=%d ep=%d topk=%d)",
|
||||
recv_rows,
|
||||
bound,
|
||||
max_tokens,
|
||||
get_parallel().moe_ep_size,
|
||||
topk,
|
||||
)
|
||||
return bound
|
||||
return "output" in inspect.signature(fused_moe).parameters
|
||||
|
||||
|
||||
class AiterRunnerCore(MoeRunnerCore):
|
||||
@@ -283,8 +200,6 @@ class AiterRunnerCore(MoeRunnerCore):
|
||||
|
||||
from aiter.fused_moe import fused_moe
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
a1_scale = (
|
||||
runner_input.a1_scale
|
||||
if runner_input.a1_scale is not None
|
||||
@@ -330,6 +245,15 @@ class AiterRunnerCore(MoeRunnerCore):
|
||||
extra["swiglu_limit"] = quant_info.swiglu_limit
|
||||
if self.config.no_combine:
|
||||
extra["no_combine"] = True
|
||||
elif (
|
||||
runner_input.output is not None
|
||||
and "output" not in extra
|
||||
and extra.get("stage2_scatter") is None
|
||||
and _aiter_fused_moe_supports_output()
|
||||
):
|
||||
# MORI EPv2 skips staging when this exact arena view is returned. Older
|
||||
# AITER versions, no_combine and stage2_scatter keep their old path.
|
||||
extra["output"] = runner_input.output
|
||||
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
|
||||
|
||||
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
|
||||
@@ -432,8 +356,8 @@ def pre_permute_standard_to_aiter(
|
||||
|
||||
|
||||
def _is_mori_dispatch_output(dispatch_output: Any) -> bool:
|
||||
# MoriEP{Normal,LL}DispatchOutput carry the post-mori-permute origin_topk_*
|
||||
# tensors that the standard DeepEP outputs lack.
|
||||
# MORI EP dispatch outputs carry the origin_topk_* tensors for combine,
|
||||
# which the standard DeepEP outputs lack.
|
||||
return hasattr(dispatch_output, "origin_topk_ids")
|
||||
|
||||
|
||||
@@ -486,6 +410,7 @@ def _pre_permute_deepep_to_aiter(
|
||||
a1_scale: Optional[torch.Tensor] = None
|
||||
num_local_tokens: Optional[torch.Tensor] = None
|
||||
output_dtype: Optional[torch.dtype] = None
|
||||
output: Optional[torch.Tensor] = None
|
||||
quant_type = quant_info.quant_type
|
||||
|
||||
if is_mori:
|
||||
@@ -494,21 +419,21 @@ def _pre_permute_deepep_to_aiter(
|
||||
a1_scale = dispatch_output.hidden_states_scale
|
||||
num_local_tokens = dispatch_output.num_recv_tokens_per_expert
|
||||
output_dtype = dispatch_output.out_dtype
|
||||
output = getattr(dispatch_output, "expert_output", None)
|
||||
mori_max = dispatch_output.recv_cap
|
||||
|
||||
# Truncate dispatch tensors to the configured cap; mori combine only
|
||||
# Truncate dispatch tensors to the configured cap; MORI EP combine only
|
||||
# reads [0, totalRecvTokenNum), so the truncated result needs no
|
||||
# padding back.
|
||||
mori_max = get_int_env_var("SGLANG_MORI_MOE_MAX_INPUT_TOKENS", 0)
|
||||
if mori_max <= 0:
|
||||
mori_max = _mori_decode_recv_bound(
|
||||
hidden_states.shape[0], topk_ids.shape[-1]
|
||||
)
|
||||
if mori_max > 0:
|
||||
# padding back. Slice the optional EPv2 output to the same row count,
|
||||
# retaining the physical arena and routing stride.
|
||||
if 0 < mori_max < hidden_states.shape[0]:
|
||||
hidden_states = hidden_states[:mori_max]
|
||||
if a1_scale is not None:
|
||||
a1_scale = a1_scale[:mori_max]
|
||||
topk_ids = topk_ids[:mori_max]
|
||||
topk_weights = topk_weights[:mori_max]
|
||||
if output is not None:
|
||||
output = output[:mori_max]
|
||||
|
||||
# Upscale dispatched activations when there is no AITER kernel for the
|
||||
# weight/activation dtype pair.
|
||||
@@ -595,6 +520,7 @@ def _pre_permute_deepep_to_aiter(
|
||||
a1_scale=a1_scale,
|
||||
num_local_tokens=num_local_tokens,
|
||||
output_dtype=output_dtype,
|
||||
output=output,
|
||||
)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -80,6 +80,9 @@ class MoeA2ABackend(Enum):
|
||||
def is_mori(self):
|
||||
return self == MoeA2ABackend.MORI
|
||||
|
||||
def is_mori_epv2(self):
|
||||
return self.is_mori() and envs.SGLANG_MORI_EP_V2.get()
|
||||
|
||||
def is_megamoe(self):
|
||||
return self == MoeA2ABackend.MEGAMOE
|
||||
|
||||
|
||||
@@ -1449,6 +1449,11 @@ def aiter_w8a8_block_fp8_linear(
|
||||
# assert input_scale is None
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
output_dtype = torch.bfloat16 if input_scale is not None else input.dtype
|
||||
# DP-attention idle ranks run zero-token forwards (eager has no padded graph
|
||||
# bucket); CK blockscale GEMM fails M == 0 with invalid configuration argument.
|
||||
if input_2d.shape[0] == 0:
|
||||
return input.new_empty(output_shape, dtype=output_dtype)
|
||||
|
||||
# dense linears converted from MXFP8 carry a rowwise-fp8 copy; its ptpc GEMM
|
||||
# beats the block-fp8 GEMMs at decode-sized M
|
||||
@@ -1524,15 +1529,13 @@ def aiter_w8a8_block_fp8_linear(
|
||||
weight,
|
||||
x_scale,
|
||||
weight_scale,
|
||||
dtype=torch.bfloat16 if input_scale is not None else input.dtype,
|
||||
dtype=output_dtype,
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
output += bias
|
||||
|
||||
return output.to(
|
||||
dtype=torch.bfloat16 if input_scale is not None else input_2d.dtype
|
||||
).view(*output_shape)
|
||||
return output.to(dtype=output_dtype).view(*output_shape)
|
||||
|
||||
|
||||
def triton_w8a8_block_fp8_linear(
|
||||
|
||||
@@ -633,6 +633,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
set_weight_attrs(w2_weight_bias, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
|
||||
# Resolve MORI dtype overrides and build the EPv2 op before KV cache
|
||||
# sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
|
||||
layer.dispatcher.set_quant_config({})
|
||||
|
||||
if self.use_marlin and not self.use_mega_moe:
|
||||
from sglang.srt.layers.quantization.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
@@ -1879,6 +1884,11 @@ class Mxfp4DynamicQuantMoEMethod(FusedMoEMethodBase):
|
||||
return w, mx_scales
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
|
||||
# Resolve MORI dtype overrides and build the EPv2 op before KV cache
|
||||
# sizing. No weight_dtype keeps bf16 dispatch, the validated default here.
|
||||
layer.dispatcher.set_quant_config({})
|
||||
|
||||
w13, w13_mx_scales = self.mxfp4_quantize(layer.w13_weight.data)
|
||||
w2, w2_mx_scales = self.mxfp4_quantize(layer.w2_weight.data)
|
||||
|
||||
|
||||
@@ -796,6 +796,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
|
||||
):
|
||||
layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"})
|
||||
|
||||
if get_moe_a2a_backend().is_mori() and hasattr(layer, "dispatcher"):
|
||||
# Resolve MORI dtype overrides and build the EPv2 op before KV cache sizing.
|
||||
layer.dispatcher.set_quant_config({"weight_dtype": layer.w13_weight.dtype})
|
||||
|
||||
if self.use_flashinfer_cutlass:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
materialize_swiglu_params_for_cutlass,
|
||||
|
||||
@@ -79,6 +79,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
_tbo_event,
|
||||
attn_tp_all_gather,
|
||||
attn_tp_all_gather_into_tensor,
|
||||
attn_tp_all_reduce,
|
||||
dp_gather_partial,
|
||||
dp_gather_replicate,
|
||||
@@ -2676,6 +2677,28 @@ def _every_row_routed(forward_batch: ForwardBatch, num_rows: int):
|
||||
) = saved
|
||||
|
||||
|
||||
def _attn_tp_all_gather_chunks(
|
||||
local: torch.Tensor, chunk_rows: List[int]
|
||||
) -> torch.Tensor:
|
||||
# Tensor all_gather stays on aiter AG / pynccl under graph capture; the list
|
||||
# all_gather falls back to ProcessGroupNCCL, which invalidates HIP capture.
|
||||
max_rows = max(chunk_rows)
|
||||
padded = local
|
||||
if local.shape[0] != max_rows:
|
||||
padded = local.new_zeros((max_rows, *local.shape[1:]))
|
||||
padded[: local.shape[0]].copy_(local)
|
||||
gathered = local.new_empty((len(chunk_rows) * max_rows, *local.shape[1:]))
|
||||
attn_tp_all_gather_into_tensor(gathered, padded.contiguous())
|
||||
if all(rows == max_rows for rows in chunk_rows):
|
||||
return gathered
|
||||
return torch.cat(
|
||||
[
|
||||
gathered[i * max_rows : i * max_rows + rows]
|
||||
for i, rows in enumerate(chunk_rows)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -3556,9 +3579,14 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
hidden_states = hidden_states + _shared_local[:n]
|
||||
if _use_tp_attn_a2a_scatter:
|
||||
assert _a2a_scatter_chunks is not None
|
||||
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
|
||||
attn_tp_all_gather(gathered, hidden_states.contiguous())
|
||||
hidden_states = torch.cat(gathered)
|
||||
if _is_hip:
|
||||
hidden_states = _attn_tp_all_gather_chunks(
|
||||
hidden_states, chunk_rows=[t.shape[0] for t in _a2a_scatter_chunks]
|
||||
)
|
||||
else:
|
||||
gathered = [torch.empty_like(t) for t in _a2a_scatter_chunks]
|
||||
attn_tp_all_gather(gathered, hidden_states.contiguous())
|
||||
hidden_states = torch.cat(gathered)
|
||||
return hidden_states
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvPro
|
||||
from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config
|
||||
from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
@@ -951,8 +952,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
input_embeds = self.forward_embed(input_ids)
|
||||
x = input_embeds
|
||||
pre = None
|
||||
for stage in self.stages:
|
||||
x, pre = stage(positions, x, forward_batch, pre)
|
||||
# Draft MoE layers have no recorder layer index, so keep them out of the stats.
|
||||
with get_global_expert_distribution_recorder().disable_this_region():
|
||||
for stage in self.stages:
|
||||
x, pre = stage(positions, x, forward_batch, pre)
|
||||
if self.hc_pre_from_prev_sublayer:
|
||||
from sglang.kernels.ops.layernorm.mhc import hc_combine
|
||||
|
||||
|
||||
@@ -4145,8 +4145,8 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
return True
|
||||
elif moe_a2a_backend.is_none():
|
||||
return True
|
||||
elif moe_a2a_backend.is_flashinfer():
|
||||
# FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank
|
||||
elif moe_a2a_backend.is_flashinfer() or moe_a2a_backend.is_mori_epv2():
|
||||
# FlashInfer/MORI EPv2 A2A need a rank-invariant, DP-synchronized per-rank
|
||||
# token count: MoeAlltoAll uses fixed-geometry buffers and the decode
|
||||
# cuda-graph bucket must be identical across EP ranks, otherwise ranks
|
||||
# replay different-sized graphs -> geometry mismatch -> illegal memory
|
||||
@@ -4155,9 +4155,7 @@ def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
|
||||
# max-based graph bucket). See #30432 re: the misleading flag name.
|
||||
return True
|
||||
elif moe_a2a_backend.is_mori() and get_bool_env_var(
|
||||
"SGLANG_MORI_RECV_BOUND", "false"
|
||||
):
|
||||
elif moe_a2a_backend.is_mori() and envs.SGLANG_MORI_RECV_BOUND.get():
|
||||
# Same bookkeeping, for the same reason. Bounding mori's receive
|
||||
# buffer means baking a fan-in size into a captured graph, and the
|
||||
# fan-in depends on what the *peers* send. Without a DP-synchronized
|
||||
|
||||
Reference in New Issue
Block a user