[Refactor] moe: Scope shared-expert configuration to each model (#44913)
This commit is contained in:
@@ -39,6 +39,12 @@ class Model(msgspec.Struct):
|
||||
doc="Whether weight loading overlaps startup.",
|
||||
)
|
||||
|
||||
# Runtime-only binding: loaders/runners scope the immutable model-owned value.
|
||||
shared_experts = Derived(
|
||||
fn="sglang.srt.configs.shared_experts.default_shared_experts_config",
|
||||
doc="Shared-expert layout of the model being constructed or executed.",
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Model and tokenizer
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Resolved shared-expert layout for one loaded model."""
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class SharedExpertsConfig(msgspec.Struct, frozen=True):
|
||||
"""Immutable layout shared by a model, its routers and its captured graphs."""
|
||||
|
||||
fusion_disabled: bool = False
|
||||
num_fused_shared_experts: int = 0
|
||||
use_waterfill: bool = False
|
||||
|
||||
@property
|
||||
def num_shared_topk_slots(self) -> int:
|
||||
"""Shared id/weight slots emitted before Waterfill expansion."""
|
||||
return 0 if self.use_waterfill else self.num_fused_shared_experts
|
||||
|
||||
|
||||
def default_shared_experts_config(args) -> SharedExpertsConfig:
|
||||
"""Publication has no loaded model yet; retain only the startup intent."""
|
||||
return SharedExpertsConfig(fusion_disabled=args.disable_shared_experts_fusion)
|
||||
@@ -24,9 +24,8 @@ from sglang.srt.layers.moe.topk import (
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
shared_topk_slots,
|
||||
waterfill_routes_shared_experts,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model
|
||||
from sglang.srt.utils import is_hip, is_npu, is_xpu
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -41,7 +40,7 @@ class HashTopK(nn.Module):
|
||||
self,
|
||||
topk,
|
||||
num_experts,
|
||||
num_fused_shared_experts,
|
||||
*,
|
||||
vocab_size,
|
||||
scoring_func="sqrtsoftplus",
|
||||
routed_scaling_factor=1.5,
|
||||
@@ -51,20 +50,18 @@ class HashTopK(nn.Module):
|
||||
super().__init__()
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.enable_waterfill = waterfill_routes_shared_experts(
|
||||
num_fused_shared_experts
|
||||
)
|
||||
self.shared_experts = get_model().shared_experts
|
||||
self.enable_waterfill = self.shared_experts.use_waterfill
|
||||
self.waterfill_balancer = None
|
||||
num_routed_experts_per_token = topk - num_fused_shared_experts
|
||||
|
||||
self.num_experts = num_experts
|
||||
# Trailing columns the hash router fills with fused shared experts.
|
||||
self.num_shared_topk_slots = shared_topk_slots(num_fused_shared_experts)
|
||||
self.topk = num_routed_experts_per_token + self.num_shared_topk_slots
|
||||
# topk counts routed experts; the trailing columns the hash router fills
|
||||
# with fused shared experts follow them.
|
||||
self.topk = topk + self.num_shared_topk_slots
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.score_func = scoring_func
|
||||
self.tid2eid = nn.Parameter(
|
||||
torch.empty(vocab_size, num_routed_experts_per_token, dtype=torch.int32),
|
||||
torch.empty(vocab_size, topk, dtype=torch.int32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self._init_default_tid2eid()
|
||||
@@ -78,6 +75,10 @@ class HashTopK(nn.Module):
|
||||
"with fused shared experts; pass --disable-shared-experts-fusion."
|
||||
)
|
||||
|
||||
@property
|
||||
def num_shared_topk_slots(self) -> int:
|
||||
return self.shared_experts.num_shared_topk_slots
|
||||
|
||||
def _init_default_tid2eid(self) -> None:
|
||||
topk = self.tid2eid.shape[1]
|
||||
if topk == 0:
|
||||
@@ -286,7 +287,7 @@ class HashTopK(nn.Module):
|
||||
num_physical_routed_experts,
|
||||
TopKConfig(
|
||||
top_k=self.topk,
|
||||
num_shared_topk_slots=num_shared_topk_slots,
|
||||
shared_experts=self.shared_experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=self.apply_routed_scaling_factor_on_output,
|
||||
),
|
||||
|
||||
@@ -38,10 +38,12 @@ import triton.language as tl
|
||||
if TYPE_CHECKING:
|
||||
from triton_kernels.tensor_details.ragged_tensor import RaggedTensorMetadata
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_forward,
|
||||
get_lora,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_server_args,
|
||||
)
|
||||
@@ -106,8 +108,6 @@ from sglang.srt.layers.moe import (
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
has_per_rank_fused_shared_slots,
|
||||
shared_topk_slots,
|
||||
waterfill_routes_shared_experts,
|
||||
)
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import (
|
||||
@@ -258,9 +258,7 @@ class TopKConfig:
|
||||
topk_group: Optional[int] = None
|
||||
num_expert_group: Optional[int] = None
|
||||
renormalize: bool = True
|
||||
# Trailing columns of top_k that the router fills with fused shared experts.
|
||||
# Zero when Waterfill appends the shared expert after routing.
|
||||
num_shared_topk_slots: int = 0
|
||||
shared_experts: SharedExpertsConfig = SharedExpertsConfig()
|
||||
custom_routing_function: Optional[Callable] = None
|
||||
correction_bias: Optional[torch.Tensor] = None
|
||||
torch_native: bool = False
|
||||
@@ -285,6 +283,10 @@ class TopKConfig:
|
||||
default=None, init=False, repr=False, compare=False
|
||||
)
|
||||
|
||||
@property
|
||||
def num_shared_topk_slots(self) -> int:
|
||||
return self.shared_experts.num_shared_topk_slots
|
||||
|
||||
def correction_bias_for_dtype(self, dtype: torch.dtype) -> Optional[torch.Tensor]:
|
||||
"""Return correction bias in ``dtype``, reusing a per-TopK lazy copy."""
|
||||
correction_bias = self.correction_bias
|
||||
@@ -588,12 +590,11 @@ def _simulate_balanced_routing(
|
||||
class TopK(BaseFusedOp):
|
||||
"""
|
||||
Parameters:
|
||||
--top_k: The all number of top experts selected per token, including the fused shared expert(s).
|
||||
--top_k: the number of routed experts selected per token. When the model being built
|
||||
fuses shared experts (``get_num_fused_shared_experts``), their columns follow: written
|
||||
by the router (``TopKConfig.num_shared_topk_slots``), or appended by Waterfill.
|
||||
--layer_id: global index of the owning layer. Required: routed-experts capture and
|
||||
per-layer EPLB state are keyed by it.
|
||||
--num_fused_shared_experts: shared experts the experts layer runs as extra expert slots.
|
||||
Their trailing top-k columns are written by the router, or by Waterfill when it is
|
||||
enabled (``TopKConfig.num_shared_topk_slots`` then excludes them).
|
||||
--routed_scaling_factor: the scaling factor for routed experts in topk_weights.
|
||||
--fused_shared_experts_scaling_factor: absolute appended weight on the global AITER
|
||||
path; otherwise multiplies the router's existing shared weight under standard EP.
|
||||
@@ -609,7 +610,6 @@ class TopK(BaseFusedOp):
|
||||
topk_group: Optional[int] = None,
|
||||
num_expert_group: Optional[int] = None,
|
||||
renormalize: bool = True,
|
||||
num_fused_shared_experts: int = 0,
|
||||
custom_routing_function: Optional[Callable] = None,
|
||||
scoring_func: str = "softmax",
|
||||
correction_bias: Optional[torch.Tensor] = None,
|
||||
@@ -632,23 +632,21 @@ class TopK(BaseFusedOp):
|
||||
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.enable_waterfill = waterfill_routes_shared_experts(
|
||||
num_fused_shared_experts
|
||||
)
|
||||
shared_experts = get_model().shared_experts
|
||||
self.enable_waterfill = shared_experts.use_waterfill
|
||||
self.waterfill_balancer = None
|
||||
num_shared_slots = shared_topk_slots(num_fused_shared_experts)
|
||||
num_routed_experts_per_token = top_k - num_fused_shared_experts
|
||||
num_shared_slots = shared_experts.num_shared_topk_slots
|
||||
|
||||
# Under the flashinfer_mxfp4 backend, fp4-expert ckpts take STANDARD
|
||||
# (consumes topk_ids/weights); otherwise BYPASSED. No-op on other backends.
|
||||
self.is_fp4_experts = is_fp4_experts
|
||||
self.topk_config = TopKConfig(
|
||||
top_k=num_routed_experts_per_token + num_shared_slots,
|
||||
top_k=top_k + num_shared_slots,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
num_shared_topk_slots=num_shared_slots,
|
||||
shared_experts=shared_experts,
|
||||
custom_routing_function=custom_routing_function,
|
||||
correction_bias=correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
|
||||
@@ -9,6 +9,7 @@ from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
@@ -500,12 +501,6 @@ def initialize_moe_config():
|
||||
moe.tbo_token_distribution_threshold = overlap.tbo_token_distribution_threshold
|
||||
moe.disable_fp4_allgather = exec_moe.disable_flashinfer_cutlass_moe_fp4_allgather
|
||||
moe.quantization = get_model().quantization
|
||||
# Seeded with the user's intent; each model's gate refines the ACTIVE
|
||||
# value for its own build (install_shared_experts_fusion_decision).
|
||||
moe.disable_shared_experts_fusion = exec_moe.disable_shared_experts_fusion
|
||||
moe.speculative_disable_shared_experts_fusion = (
|
||||
exec_moe.disable_shared_experts_fusion
|
||||
)
|
||||
|
||||
|
||||
def get_moe_a2a_backend() -> MoeA2ABackend:
|
||||
@@ -543,29 +538,11 @@ def get_speculative_moe_a2a_backend() -> MoeA2ABackend:
|
||||
|
||||
|
||||
def is_shared_experts_fusion_disabled() -> bool:
|
||||
"""The ACTIVE shared-experts-fusion decision for the model being built.
|
||||
return get_model().shared_experts.fusion_disabled
|
||||
|
||||
Written (both ways) by each MoE model's gate before its layers construct;
|
||||
falls back to the config intent when no gate has run (models without an
|
||||
auto-disable gate read the intent directly off the bag instead).
|
||||
|
||||
Construction-time only: a forward reads what its build baked in
|
||||
(``num_fused_shared_experts`` on the layer). During a draft's build this
|
||||
flag holds the DRAFT's decision, so a forward-time read would race the
|
||||
build window — refuse it loudly."""
|
||||
from sglang.srt.model_executor.forward_context import has_forward_context
|
||||
|
||||
if has_forward_context():
|
||||
raise AssertionError(
|
||||
"is_shared_experts_fusion_disabled() called inside a forward: the "
|
||||
"fusion decision is construction-time state (it can hold the draft's "
|
||||
"value while a draft builds). Read the value your build baked in, "
|
||||
"e.g. the layer's num_fused_shared_experts."
|
||||
)
|
||||
moe = get_flags().moe
|
||||
if moe.disable_shared_experts_fusion is None:
|
||||
return get_exec().moe.disable_shared_experts_fusion
|
||||
return moe.disable_shared_experts_fusion
|
||||
def get_num_fused_shared_experts() -> int:
|
||||
return get_model().shared_experts.num_fused_shared_experts
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -574,9 +551,6 @@ def draft_model_build_scope():
|
||||
settings; every value below returns to the target's on exit, including on
|
||||
error.
|
||||
|
||||
- Shared-experts fusion: the gates the draft runs also record their
|
||||
decision on the speculative leaf; the target's ACTIVE
|
||||
``disable_shared_experts_fusion`` is restored.
|
||||
- ``boundary_reduction`` is ``--speculative-boundary-reduction``.
|
||||
Boundaries built here keep the resolved value after exit.
|
||||
- ``enable_w4a4_mxfp4_megamoe`` is ``--speculative-enable-w4a4-mxfp4-megamoe``
|
||||
@@ -588,7 +562,6 @@ def draft_model_build_scope():
|
||||
whole lifecycle (build + capture + forward), which not every worker does.
|
||||
"""
|
||||
moe = get_flags().moe
|
||||
original_fusion = moe.disable_shared_experts_fusion
|
||||
original_scope = moe.in_speculative_scope
|
||||
try:
|
||||
moe.in_speculative_scope = True
|
||||
@@ -603,7 +576,6 @@ def draft_model_build_scope():
|
||||
yield
|
||||
finally:
|
||||
moe.in_speculative_scope = original_scope
|
||||
moe.disable_shared_experts_fusion = original_fusion
|
||||
|
||||
|
||||
def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
|
||||
@@ -615,36 +587,29 @@ def _draft_w4a4_mxfp4_megamoe_override() -> AbstractContextManager:
|
||||
return get_exec().moe.override(enable_w4a4_mxfp4_megamoe=draft_w4a4)
|
||||
|
||||
|
||||
def install_shared_experts_fusion_decision(
|
||||
def resolve_shared_experts_config(
|
||||
model_class, hf_config, quant_config
|
||||
) -> None:
|
||||
"""Decide whether this runner's model fuses its shared experts, and install
|
||||
the answer for the model it is about to build.
|
||||
|
||||
Called from the loader's single model-instantiation point, so the decision
|
||||
is made once per runner — before any layer exists — and the model classes
|
||||
are pure readers (``is_shared_experts_fusion_disabled``). A model family
|
||||
that can auto-disable exposes the conditions as
|
||||
``shared_experts_fusion_disable_reason(hf_config, quant_config)``; families
|
||||
without one follow the user's intent.
|
||||
|
||||
Inside ``draft_model_build_scope`` the answer also lands on the speculative
|
||||
leaf, so a flags dump afterwards shows both runners' decisions.
|
||||
"""
|
||||
|
||||
) -> SharedExpertsConfig:
|
||||
"""Resolve a model layout before construction; fusion requires both hooks."""
|
||||
disabled = get_exec().moe.disable_shared_experts_fusion
|
||||
gate = getattr(model_class, "shared_experts_fusion_disable_reason", None)
|
||||
if not disabled:
|
||||
gate = getattr(model_class, "shared_experts_fusion_disable_reason", None)
|
||||
reason = gate(hf_config, quant_config) if gate is not None else None
|
||||
if reason:
|
||||
log_info_on_rank0(
|
||||
logger, f"{reason} Shared experts fusion optimization is disabled."
|
||||
)
|
||||
disabled = True
|
||||
moe = get_flags().moe
|
||||
moe.disable_shared_experts_fusion = disabled
|
||||
if moe.in_speculative_scope:
|
||||
moe.speculative_disable_shared_experts_fusion = disabled
|
||||
count_fusable = getattr(model_class, "num_fusable_shared_experts", None)
|
||||
if disabled or gate is None or count_fusable is None:
|
||||
num_fused = 0
|
||||
else:
|
||||
num_fused = count_fusable(hf_config)
|
||||
return SharedExpertsConfig(
|
||||
fusion_disabled=disabled,
|
||||
num_fused_shared_experts=num_fused,
|
||||
use_waterfill=num_fused > 0 and get_exec().moe.enable_waterfill,
|
||||
)
|
||||
|
||||
|
||||
def get_deepep_mode() -> DeepEPMode:
|
||||
@@ -699,24 +664,6 @@ def has_per_rank_fused_shared_slots(num_shared: int) -> bool:
|
||||
return num_shared > 0 and uses_per_rank_fused_shared_slots()
|
||||
|
||||
|
||||
def waterfill_routes_shared_experts(num_fused_shared_experts: int) -> bool:
|
||||
"""Whether Waterfill, not the top-k router, writes the fused shared experts' column.
|
||||
|
||||
Waterfill appends that column after the EPLB remap and may place it on another
|
||||
rank, so the router it follows emits the routed columns only."""
|
||||
return num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill
|
||||
|
||||
|
||||
def shared_topk_slots(num_fused_shared_experts: int) -> int:
|
||||
"""How many trailing top-k columns the router fills with fused shared experts.
|
||||
|
||||
``num_fused_shared_experts`` counts the shared experts the experts layer runs as
|
||||
extra expert slots; the router writes their columns unless Waterfill does."""
|
||||
if waterfill_routes_shared_experts(num_fused_shared_experts):
|
||||
return 0
|
||||
return num_fused_shared_experts
|
||||
|
||||
|
||||
def is_flashinfer_cutedsl_v1_path() -> bool:
|
||||
"""CuteDSL v1 + DeepEP low-latency path (no MoeRunner, no autotune)."""
|
||||
return (
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_lora,
|
||||
get_model,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
)
|
||||
@@ -734,6 +735,12 @@ class CPUGraphRunner:
|
||||
return is_bs_supported
|
||||
|
||||
def capture(self) -> None:
|
||||
with get_model().override(
|
||||
shared_experts=self.model_runner.model.shared_experts_config
|
||||
):
|
||||
self._capture()
|
||||
|
||||
def _capture(self) -> None:
|
||||
capture_range = (
|
||||
tqdm.tqdm(list(reversed(self.capture_bs)))
|
||||
if get_parallel().tp_rank == 0
|
||||
|
||||
@@ -1794,7 +1794,10 @@ class ModelRunner:
|
||||
forward_batch.split_index + forward_count,
|
||||
self.model_config.num_hidden_layers,
|
||||
)
|
||||
with device_timer_ctx(self.device_timer, "split_prefill"):
|
||||
with (
|
||||
get_model().override(shared_experts=self.model.shared_experts_config),
|
||||
device_timer_ctx(self.device_timer, "split_prefill"),
|
||||
):
|
||||
ret = self.model.forward_split_prefill(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
@@ -1968,7 +1971,10 @@ class ModelRunner:
|
||||
ctx_mgr = contextlib.nullcontext()
|
||||
else:
|
||||
ctx_mgr = forward_context(ForwardContext(attn_backend=self.attn_backend))
|
||||
with ctx_mgr:
|
||||
with (
|
||||
get_model().override(shared_experts=self.model.shared_experts_config),
|
||||
ctx_mgr,
|
||||
):
|
||||
mode_check = (
|
||||
forward_batch.forward_mode.is_cpu_graph
|
||||
if self.device == "cpu"
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_model,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import create_dummy_verify_input
|
||||
@@ -243,7 +244,17 @@ class BaseRunner(ABC):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.tbo_plugin = TboCudaGraphRunnerPlugin()
|
||||
|
||||
def model_context(self):
|
||||
"""Select the layout owned by this runner's current model."""
|
||||
return get_model().override(
|
||||
shared_experts=self.model_runner.model.shared_experts_config
|
||||
)
|
||||
|
||||
def warmup(self) -> None:
|
||||
with self.model_context():
|
||||
self._warmup()
|
||||
|
||||
def _warmup(self) -> None:
|
||||
"""Run kernel warmup + autotune once, gated by mr._kernel_warmed_up."""
|
||||
mr = self.model_runner
|
||||
if getattr(mr, "_kernel_warmed_up", False):
|
||||
|
||||
@@ -1107,6 +1107,10 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return forward_batch, attn_backend, pp_proxy_tensors
|
||||
|
||||
def capture(self) -> None:
|
||||
with self.model_context():
|
||||
self._capture()
|
||||
|
||||
def _capture(self) -> None:
|
||||
# Warm up + autotune kernels once before capture (run-once across the
|
||||
# decode + prefill runners; see BaseRunner.warmup).
|
||||
self.warmup()
|
||||
@@ -1593,7 +1597,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
shared_read_ends = self._resolve_shared_read_ends(
|
||||
self._replay_attn_backend(), forward_batch.forward_mode
|
||||
)
|
||||
with timer_ctx, self.backend.replay_session():
|
||||
with self.model_context(), timer_ctx, self.backend.replay_session():
|
||||
self.load_batch(forward_batch, pp_proxy_tensors)
|
||||
if envs.SGLANG_LOG_DECODE_GRAPH_KEY.get():
|
||||
logger.info(
|
||||
|
||||
@@ -209,20 +209,21 @@ class EagerRunner(BaseRunner):
|
||||
def execute(
|
||||
self, forward_batch: ForwardBatch, pp_proxy_tensors=None, **kwargs
|
||||
) -> Any:
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_mixed() and not is_npu() and get_cp_strategy() is None:
|
||||
# A mixed batch is extend-shaped (decode tails are 1-token
|
||||
# extends); run it as EXTEND. NPU keeps MIXED for its dedicated
|
||||
# kernel; CP keeps it to skip the zigzag split.
|
||||
forward_batch.forward_mode = ForwardMode.EXTEND
|
||||
mode = ForwardMode.EXTEND
|
||||
if mode.is_decode():
|
||||
return self._execute_decode(forward_batch, pp_proxy_tensors)
|
||||
if mode.is_idle():
|
||||
return self._execute_idle(forward_batch, pp_proxy_tensors)
|
||||
if mode.is_extend(include_draft_extend_v2=True):
|
||||
return self._execute_extend(forward_batch, pp_proxy_tensors)
|
||||
raise ValueError(f"Invalid forward mode for eager runner: {mode}")
|
||||
with self.model_context():
|
||||
mode = forward_batch.forward_mode
|
||||
if mode.is_mixed() and not is_npu() and get_cp_strategy() is None:
|
||||
# A mixed batch is extend-shaped (decode tails are 1-token
|
||||
# extends); run it as EXTEND. NPU keeps MIXED for its dedicated
|
||||
# kernel; CP keeps it to skip the zigzag split.
|
||||
forward_batch.forward_mode = ForwardMode.EXTEND
|
||||
mode = ForwardMode.EXTEND
|
||||
if mode.is_decode():
|
||||
return self._execute_decode(forward_batch, pp_proxy_tensors)
|
||||
if mode.is_idle():
|
||||
return self._execute_idle(forward_batch, pp_proxy_tensors)
|
||||
if mode.is_extend(include_draft_extend_v2=True):
|
||||
return self._execute_extend(forward_batch, pp_proxy_tensors)
|
||||
raise ValueError(f"Invalid forward mode for eager runner: {mode}")
|
||||
|
||||
def _resolve_decode_pdmux(
|
||||
self,
|
||||
|
||||
@@ -1657,6 +1657,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
return forward_batch, self.model_runner.attn_backend
|
||||
|
||||
def capture(self) -> None:
|
||||
with self.model_context():
|
||||
self._capture()
|
||||
|
||||
def _capture(self) -> None:
|
||||
# Warm up + autotune kernels once before capture (run-once across the
|
||||
# decode + prefill runners; see BaseRunner.warmup).
|
||||
self.warmup()
|
||||
@@ -2307,7 +2311,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
self, forward_batch: ForwardBatch, **kwargs
|
||||
) -> Optional[Union[LogitsProcessorOutput, PPProxyTensors, EmbeddingPoolerOutput]]:
|
||||
self._validate_capture_hidden_mode(forward_batch)
|
||||
with self.backend.replay_session():
|
||||
with self.model_context(), self.backend.replay_session():
|
||||
static_forward_batch = self.load_batch(forward_batch, **kwargs)
|
||||
static_num_tokens = len(static_forward_batch.input_ids)
|
||||
raw_num_tokens = self.raw_num_tokens
|
||||
|
||||
@@ -86,7 +86,7 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
|
||||
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
install_shared_experts_fusion_decision,
|
||||
resolve_shared_experts_config,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
@@ -285,11 +285,7 @@ def _initialize_model(
|
||||
) -> nn.Module:
|
||||
"""Initialize a model with the given configurations."""
|
||||
model_class, _ = get_model_architecture(model_config)
|
||||
# Decide the shared-experts-fusion question here, once per runner, before any
|
||||
# layer exists: this is the only place a model class is instantiated, and it
|
||||
# is the last point that still knows both the checkpoint's quantization and
|
||||
# (through the build scope) whether this runner is a draft.
|
||||
install_shared_experts_fusion_decision(
|
||||
shared_experts = resolve_shared_experts_config(
|
||||
model_class, model_config.hf_config, quant_config
|
||||
)
|
||||
kwargs = {
|
||||
@@ -305,8 +301,10 @@ def _initialize_model(
|
||||
if load_config.draft_model_idx is not None:
|
||||
kwargs["draft_model_idx"] = load_config.draft_model_idx
|
||||
|
||||
model = model_class(**kwargs)
|
||||
check_stage_producers(model)
|
||||
with get_model().override(shared_experts=shared_experts):
|
||||
model = model_class(**kwargs)
|
||||
check_stage_producers(model)
|
||||
model.shared_experts_config = shared_experts
|
||||
return model
|
||||
|
||||
|
||||
|
||||
@@ -68,6 +68,10 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
|
||||
hf_config.text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config.text_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -71,7 +72,6 @@ class BailingMoEModelNextN(nn.Module):
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
num_fused_shared_experts: int = 0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.layer_group_size = 1
|
||||
@@ -122,9 +122,6 @@ class BailingMoEModelNextN(nn.Module):
|
||||
}
|
||||
if _is_bailing_moe_v3_config(config):
|
||||
decoder_layer_cls = BailingMoeV3DecoderLayer
|
||||
decoder_kwargs["num_fused_shared_experts"] = (
|
||||
num_fused_shared_experts
|
||||
)
|
||||
self.decoder = decoder_layer_cls(config, **decoder_kwargs)
|
||||
else:
|
||||
self.decoder = BailingMoEBlock(
|
||||
@@ -215,6 +212,12 @@ class BailingMoeForCausalLMNextN(nn.Module):
|
||||
expected_architecture="BailingMoeForCausalLMNextN",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
if not _is_bailing_moe_v3_config(hf_config):
|
||||
return 0
|
||||
return BailingMoeV3ForCausalLM.num_fusable_shared_experts(hf_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
@@ -224,19 +227,16 @@ class BailingMoeForCausalLMNextN(nn.Module):
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
is_bailing_moe_v3 = _is_bailing_moe_v3_config(config)
|
||||
if is_bailing_moe_v3:
|
||||
BailingMoeV3ForCausalLM.determine_num_fused_shared_experts(self)
|
||||
elif hasattr(self, "determine_num_fused_shared_experts"):
|
||||
# Asystem has determine_num_fused_shared_experts but theta does not.
|
||||
self.determine_num_fused_shared_experts("BailingMoeForCausalLMNextN")
|
||||
BailingMoeV3ForCausalLM.log_shared_experts_fusion(self)
|
||||
|
||||
self.model = BailingMoEModelNextN(
|
||||
config,
|
||||
quant_config,
|
||||
prefix=add_prefix("model", prefix),
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
|
||||
@@ -48,7 +48,7 @@ from sglang.srt.layers.moe import (
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.multi_gate import (
|
||||
create_multi_gate_mm_indices,
|
||||
multi_gate_triton_kernel,
|
||||
@@ -445,7 +445,6 @@ class BailingMoE(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
layer_id: int = 0,
|
||||
prefix: str = "moe",
|
||||
num_fused_shared_experts: int = 0,
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
):
|
||||
super().__init__()
|
||||
@@ -466,7 +465,7 @@ class BailingMoE(nn.Module):
|
||||
self.score_function = getattr(config, "score_function", None)
|
||||
self.multi_gate = is_bailing_multi_gate_enabled(config)
|
||||
|
||||
self.num_fused_shared_experts = num_fused_shared_experts
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
expert_swiglu_limit_list = getattr(config, "expert_swiglu_limit_list", None)
|
||||
share_expert_swiglu_limit_list = getattr(
|
||||
@@ -562,8 +561,7 @@ class BailingMoE(nn.Module):
|
||||
# Non-A2A EP (standard): each GPU computes shared expert, outputs are summed
|
||||
# via all_reduce → scale down by 1/ep_size to avoid double counting.
|
||||
# Note: A2A EP (DeepEP) + fused shared experts is impossible (expert routing
|
||||
# breaks for the extra shared expert ID), so it's auto-disabled in
|
||||
# determine_num_fused_shared_experts().
|
||||
# breaks for the extra shared expert ID), so the fusion veto disables it.
|
||||
fused_shared_experts_scaling_factor = None
|
||||
if self.moe_ep_size > 1 and self.num_fused_shared_experts > 0:
|
||||
fused_shared_experts_scaling_factor = 1.0 / float(self.moe_ep_size)
|
||||
@@ -581,7 +579,7 @@ class BailingMoE(nn.Module):
|
||||
gemm1_clamp_limit=self.expert_swiglu_limit,
|
||||
)
|
||||
self.topk = TopK(
|
||||
top_k=self.top_k + self.num_fused_shared_experts,
|
||||
top_k=self.top_k,
|
||||
layer_id=self.layer_id,
|
||||
use_grouped_topk=self.use_grouped_topk,
|
||||
renormalize=self.norm_expert_prob,
|
||||
@@ -594,7 +592,6 @@ class BailingMoE(nn.Module):
|
||||
apply_routed_scaling_factor_on_output=(
|
||||
self.experts.should_fuse_routed_scaling_factor_in_topk
|
||||
),
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
fused_shared_experts_scaling_factor=fused_shared_experts_scaling_factor,
|
||||
is_fp4_experts=(
|
||||
quant_config is not None
|
||||
@@ -1016,7 +1013,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
layer_id: int = 0,
|
||||
prefix: str = "layer",
|
||||
num_fused_shared_experts: int = 0,
|
||||
is_nextn: bool = False,
|
||||
alt_stream: Optional[torch.cuda.Stream] = None,
|
||||
) -> None:
|
||||
@@ -1094,7 +1090,6 @@ class BailingMoELinearDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
layer_id=self.layer_id,
|
||||
prefix=prefix,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
alt_stream=alt_stream,
|
||||
)
|
||||
else:
|
||||
@@ -1210,7 +1205,6 @@ class BailingMoELinearModel(nn.Module):
|
||||
config: PretrainedConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
num_fused_shared_experts: int = 0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pp_group = get_parallel().pp_group
|
||||
@@ -1258,7 +1252,6 @@ class BailingMoELinearModel(nn.Module):
|
||||
decoder_kwargs = {
|
||||
"quant_config": quant_config,
|
||||
"layer_id": layer_idx,
|
||||
"num_fused_shared_experts": num_fused_shared_experts,
|
||||
}
|
||||
return BailingMoELinearDecoderLayer(
|
||||
layer_config,
|
||||
@@ -1394,14 +1387,14 @@ class BailingMoeV3ForCausalLM(nn.Module):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
# Determine num_fused_shared_experts
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.log_shared_experts_fusion()
|
||||
|
||||
self.model = BailingMoELinearModel(
|
||||
self.config,
|
||||
quant_config,
|
||||
prefix=add_prefix("model", prefix),
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
@@ -1522,26 +1515,25 @@ class BailingMoeV3ForCausalLM(nn.Module):
|
||||
)
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
self.num_fused_shared_experts = (
|
||||
0
|
||||
if is_shared_experts_fusion_disabled()
|
||||
else _get_bailing_num_shared_experts(self.config)
|
||||
)
|
||||
if self.num_fused_shared_experts == 0:
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
num_shared_experts = _get_bailing_num_shared_experts(hf_config)
|
||||
# Safety check: current CUDA implementation only supports num_fused_shared_experts == 1.
|
||||
# The grouped_topk_gpu and _post_process_topk_ids functions only handle the last column,
|
||||
# which is incorrect when num_fused_shared_experts > 1.
|
||||
# AMD platform with aiter handles this correctly via fused_append_shared_experts kernel.
|
||||
if self.num_fused_shared_experts > 1 and not _is_hip:
|
||||
if num_shared_experts > 1 and not _is_hip:
|
||||
raise ValueError(
|
||||
f"num_fused_shared_experts > 1 ({self.num_fused_shared_experts}) is not "
|
||||
f"num_fused_shared_experts > 1 ({num_shared_experts}) is not "
|
||||
f"supported on CUDA platform. The current TopK implementation only handles "
|
||||
f"one fused shared expert. AMD platform with aiter supports multiple shared experts."
|
||||
)
|
||||
return num_shared_experts
|
||||
|
||||
def log_shared_experts_fusion(self):
|
||||
if self.num_fused_shared_experts == 0:
|
||||
return
|
||||
moe_ep_size = get_parallel().moe_ep_size
|
||||
if moe_ep_size > 1:
|
||||
log_info_on_rank0(
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization import Fp8Config
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -284,7 +285,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM):
|
||||
self.quant_config = quant_config
|
||||
# if not set, model load will be broken in DeepseekV3ForCausalLM load_weights()
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
nextn_quant_config = self._resolve_nextn_quant_config(config, quant_config)
|
||||
|
||||
self.model = DeepseekModelNextN(
|
||||
|
||||
@@ -1421,21 +1421,33 @@ def build_qwen2_decoder_as_encoder(
|
||||
return decoder_as_encoder
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
text_config = hf_config.text_config
|
||||
if is_ocr2_config(hf_config) or not (
|
||||
text_config.topk_method == "noaux_tc" or text_config.use_mla
|
||||
):
|
||||
return None
|
||||
return text_config
|
||||
|
||||
|
||||
class DeepseekOCRForCausalLM(nn.Module):
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
text_config = hf_config.text_config
|
||||
# Class-level hook: called before the model is built, so no `self.is_ocr2` yet.
|
||||
if is_ocr2_config(hf_config) or not (
|
||||
text_config.topk_method == "noaux_tc" or text_config.use_mla
|
||||
):
|
||||
# Those branches build the dense DeepseekForCausalLM, which has no
|
||||
# shared experts to fuse.
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -129,10 +129,10 @@ from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
adds_replicated_output_to_partial,
|
||||
filter_moe_weight_param_global_expert,
|
||||
get_num_fused_shared_experts,
|
||||
has_per_rank_fused_shared_slots,
|
||||
is_deepep_class_backend,
|
||||
is_sbo_enabled,
|
||||
is_shared_experts_fusion_disabled,
|
||||
is_tbo_enabled,
|
||||
should_add_replicated_moe_output,
|
||||
sum_post_experts_output,
|
||||
@@ -676,11 +676,13 @@ class DeepseekV2MoE(nn.Module):
|
||||
n_shared_experts = (
|
||||
0 if config.n_shared_experts is None else int(config.n_shared_experts)
|
||||
)
|
||||
_fusion_disabled = is_shared_experts_fusion_disabled()
|
||||
|
||||
# num_fused_shared_experts drives weight remapping in deepseek_weight_loader:
|
||||
# mlp.shared_experts → mlp.experts.256 when > 0.
|
||||
self.num_fused_shared_experts = 0 if _fusion_disabled else n_shared_experts
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
assert self.num_fused_shared_experts in (0, n_shared_experts), (
|
||||
f"The model fuses {self.num_fused_shared_experts} shared experts, but "
|
||||
f"this MoE block has {n_shared_experts}."
|
||||
)
|
||||
|
||||
# DeepEP and MegaMOE shared expert fusion: shared expert is fused into
|
||||
# the same MoE kernel as a local expert at each EP rank. Expert layout
|
||||
@@ -762,9 +764,8 @@ class DeepseekV2MoE(nn.Module):
|
||||
|
||||
if self.is_hash and not (is_nextn and is_deepseek_v4):
|
||||
self.topk = HashTopK(
|
||||
topk=config.num_experts_per_tok + self.num_fused_shared_experts,
|
||||
topk=config.num_experts_per_tok,
|
||||
num_experts=config.n_routed_experts,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
vocab_size=config.vocab_size,
|
||||
scoring_func=config.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
@@ -778,12 +779,11 @@ class DeepseekV2MoE(nn.Module):
|
||||
else:
|
||||
# Default: grouped noaux_tc top-k. Covers V3/V3.2/GLM-5/Glm4MoeLite.
|
||||
topk_kwargs = dict(
|
||||
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
layer_id=self.layer_id,
|
||||
renormalize=config.norm_topk_prob,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
topk_group=config.topk_group,
|
||||
scoring_func=config.scoring_func,
|
||||
correction_bias=self.gate.e_score_correction_bias,
|
||||
@@ -3282,7 +3282,8 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.use_dsa = is_deepseek_dsa(config)
|
||||
self.model = DeepseekV2Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
@@ -3330,7 +3331,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
"""Why this checkpoint cannot fuse its shared expert, or None.
|
||||
|
||||
Evaluated by the loader once per runner, before any layer is built (see
|
||||
``install_shared_experts_fusion_decision``), so it takes the config and
|
||||
``resolve_shared_experts_config``), so it takes the config and
|
||||
quantization it is asked about rather than reading an instance.
|
||||
"""
|
||||
# Need to disable if quant precision mismatch, even if
|
||||
@@ -3397,11 +3398,10 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
return "Deepseek V3/R1 W4AFP8/W4A16 model uses different quant method for routed experts and shared experts."
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Return the model's fusable shared-expert count."""
|
||||
return hf_config.n_shared_experts or 0
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.model.embed_tokens
|
||||
|
||||
@@ -90,7 +90,7 @@ from sglang.srt.layers.logits_processor import LogitsMetadata, LogitsProcessor
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
should_skip_post_experts_all_reduce,
|
||||
uses_per_rank_fused_shared_slots,
|
||||
)
|
||||
@@ -4447,7 +4447,8 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.quant_config = quant_config
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.vision = None
|
||||
if (
|
||||
config.model_type == "deepseek_v41"
|
||||
@@ -4663,11 +4664,10 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
)
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
return hf_config.n_shared_experts or 0
|
||||
|
||||
def prepare_language_model_inputs(
|
||||
self,
|
||||
|
||||
@@ -24,7 +24,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -702,6 +702,10 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
hf_config, quant_config
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
return DeepseekV4ForCausalLM.num_fusable_shared_experts(hf_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DeepSeekV4Config,
|
||||
@@ -711,9 +715,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
|
||||
)
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
dspark_config = parse_dspark_draft_config(draft_hf_config=config)
|
||||
if not dspark_config.require_markov():
|
||||
|
||||
@@ -24,7 +24,10 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.modelslim.modelslim import ModelSlimConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
@@ -264,7 +267,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.quant_config = quant_config
|
||||
self.wo_a_fp8 = wo_a_fp8_gemm_enabled(quant_config)
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
self.model = DeepseekV4ModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -156,17 +156,30 @@ class DeepseekVL2MlpProjector(nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
language_config = hf_config.language_config
|
||||
return language_config if language_config.use_mla else None
|
||||
|
||||
|
||||
class DeepseekVL2ForCausalLM(nn.Module):
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
language_config = hf_config.language_config
|
||||
if not language_config.use_mla:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
# The language model is built without a quantization config.
|
||||
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
language_config, None
|
||||
text_config, None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DeepseekVL2Config,
|
||||
|
||||
@@ -80,7 +80,7 @@ from sglang.srt.layers.moe import (
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_topk
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.fp8_utils import (
|
||||
block_quant_dequant,
|
||||
@@ -278,9 +278,7 @@ class Dots3MoE(nn.Module):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
|
||||
)
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.config = config
|
||||
self.layer_id = layer_id
|
||||
self.alt_stream = alt_stream
|
||||
@@ -314,12 +312,11 @@ class Dots3MoE(nn.Module):
|
||||
)
|
||||
|
||||
self.topk = TopK(
|
||||
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
layer_id=self.layer_id,
|
||||
renormalize=config.norm_topk_prob,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
topk_group=config.topk_group,
|
||||
correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
@@ -1605,7 +1602,8 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.model = Dots3Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -1649,10 +1647,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
)
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
return hf_config.n_shared_experts or 0
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.model.embed_tokens
|
||||
@@ -2478,6 +2476,10 @@ class DotsNoteOmniForConditionalGeneration(nn.Module):
|
||||
hf_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
return Dots3LanguageModelForCausalLM.num_fusable_shared_experts(hf_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
|
||||
@@ -14,6 +14,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -158,7 +159,7 @@ class Dots3NoteForCausalLMNextN(Dots3LanguageModelForCausalLM):
|
||||
"g_proj",
|
||||
]
|
||||
}
|
||||
self.determine_num_fused_shared_experts()
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
self.model = Dot3NoteModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -38,17 +38,30 @@ from sglang.srt.runtime_context import get_parallel
|
||||
from .dots_vlm_vit import DotsVisionTransformer
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
return None if hf_config.encoder_only else hf_config.language_config
|
||||
|
||||
|
||||
class DotsVLMForCausalLM(nn.Module):
|
||||
"""DotsVLM model for sglang inference"""
|
||||
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
if hf_config.encoder_only:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
hf_config.language_config, quant_config
|
||||
text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
def __init__(
|
||||
self, config: DotsVLMConfig, quant_config: Optional[QuantizationConfig] = None
|
||||
) -> None:
|
||||
|
||||
@@ -69,7 +69,7 @@ from sglang.srt.layers.moe.topk import TopK, is_routed_scaling_factor_fused_in_t
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
RoutingMethodType,
|
||||
filter_moe_weight_param_global_expert,
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
@@ -414,9 +414,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
|
||||
)
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
self.config = config
|
||||
self.layer_id = layer_id
|
||||
@@ -450,7 +448,7 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.topk = TopK(
|
||||
top_k=self.top_k + self.num_fused_shared_experts,
|
||||
top_k=self.top_k,
|
||||
layer_id=self.layer_id,
|
||||
renormalize=config.norm_topk_prob,
|
||||
use_grouped_topk=True,
|
||||
@@ -458,7 +456,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
topk_group=config.topk_group,
|
||||
correction_bias=self.gate.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
apply_routed_scaling_factor_on_output=getattr(
|
||||
self.experts, "should_fuse_routed_scaling_factor_in_topk", False
|
||||
),
|
||||
@@ -995,8 +992,8 @@ class Glm4MoeForCausalLM(nn.Module):
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.model = Glm4MoeModel(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -1033,11 +1030,10 @@ class Glm4MoeForCausalLM(nn.Module):
|
||||
return "GLM-4.5 W4AFP8 model uses different quant method for routed experts and shared experts."
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
return hf_config.n_shared_experts or 0
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.model.embed_tokens
|
||||
|
||||
@@ -66,7 +66,7 @@ from sglang.srt.layers.moe.topk import (
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
filter_moe_weight_param_global_expert,
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
@@ -185,9 +185,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
self.n_shared_experts = config.n_shared_experts
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
|
||||
)
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.config = config
|
||||
self.layer_id = layer_id
|
||||
self.alt_stream = alt_stream
|
||||
@@ -224,12 +222,11 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
)
|
||||
|
||||
self.topk = TopK(
|
||||
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
layer_id=self.layer_id,
|
||||
renormalize=config.norm_topk_prob,
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
topk_group=config.topk_group,
|
||||
correction_bias=self.gate.e_score_correction_bias,
|
||||
quant_config=quant_config,
|
||||
@@ -704,7 +701,8 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
self.model = Glm4MoeLiteModel(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -749,11 +747,10 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin):
|
||||
return "GLM-4.5 or GLM-4.6 cannot use shared experts fusion optimization under expert parallelism."
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
)
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
return hf_config.n_shared_experts or 0
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.model.embed_tokens
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -148,10 +149,8 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM):
|
||||
quant_config = None
|
||||
self.quant_config = quant_config
|
||||
|
||||
# The draft's own gate (its quantization can differ from the
|
||||
# target's); the decoder below reads the ACTIVE decision as it builds,
|
||||
# and num_fused_shared_experts drives the inherited loader's remap.
|
||||
self.determine_num_fused_shared_experts()
|
||||
# Drives the inherited loader's shared-expert remap after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
self.model = Glm4MoeLiteModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.layers.layer_boundary import layer_stack
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -129,12 +130,8 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM):
|
||||
quant_config = None
|
||||
self.quant_config = quant_config
|
||||
|
||||
# The draft's own gate: its quantization can differ from the
|
||||
# target's, and the decoder below reads the ACTIVE decision while it
|
||||
# builds. Also sets num_fused_shared_experts, which drives the
|
||||
# inherited loader's shared-expert remap.
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
# Drives the inherited loader's shared-expert remap after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
self.model = Glm4MoeModelNextN(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
|
||||
@@ -10,7 +10,7 @@ from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import is_shared_experts_fusion_disabled
|
||||
from sglang.srt.layers.moe.utils import get_num_fused_shared_experts
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
@@ -44,8 +44,10 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
|
||||
self.use_data_parallel = get_mm().mm_enable_dp_encoder
|
||||
vision_utils.update_vit_attn_dummy_heads_config(self.config)
|
||||
self.quant_config = quant_config
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
if self.num_fused_shared_experts:
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
|
||||
self.model = Glm4MoeModel(
|
||||
config,
|
||||
@@ -97,15 +99,13 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration):
|
||||
return "Shared experts fusion is not supported when Deepep MoE backend is enabled."
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
if is_shared_experts_fusion_disabled():
|
||||
return
|
||||
self.num_fused_shared_experts = self.config.n_shared_experts
|
||||
assert self.num_fused_shared_experts == 1, (
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
assert hf_config.n_shared_experts == 1, (
|
||||
"Only 1 fused shared expert is supported for Glm4vMoeForConditionalGeneration"
|
||||
)
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
return hf_config.n_shared_experts
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]], is_nextn=False):
|
||||
if is_nextn:
|
||||
|
||||
@@ -54,7 +54,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
get_moe_runner_backend,
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.utils import is_layer_skipped
|
||||
@@ -1277,7 +1277,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
self.lm_head = None
|
||||
self.logits_processor = None
|
||||
if not self.encoder_only:
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
if self.num_fused_shared_experts:
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
self.model = Glm5NextModel(
|
||||
text_config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
@@ -1491,16 +1494,14 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
return reason
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else self.config.n_shared_experts
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Return the model's fusable shared-expert count."""
|
||||
num_shared_experts = hf_config.text_config.n_shared_experts
|
||||
assert num_shared_experts == 1, (
|
||||
f"Only 1 fused shared expert is supported for {cls.__name__}"
|
||||
)
|
||||
if self.num_fused_shared_experts == 0:
|
||||
return
|
||||
assert self.num_fused_shared_experts == 1, (
|
||||
f"Only 1 fused shared expert is supported for {type(self).__name__}"
|
||||
)
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
return num_shared_experts
|
||||
|
||||
def wants_prefill_autotune(self) -> bool:
|
||||
backend = get_moe_runner_backend()
|
||||
|
||||
@@ -632,6 +632,11 @@ def mm_projection_auto(
|
||||
return projected.reshape(-1, projected.shape[-1])
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
return None if hf_config.encoder_only else hf_config.text_config
|
||||
|
||||
|
||||
class KimiK25ForConditionalGeneration(nn.Module):
|
||||
supports_cuda_vmm_feature_transport = True
|
||||
|
||||
@@ -646,12 +651,20 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
if hf_config.encoder_only:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
return DeepseekV3ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
hf_config.text_config, quant_config
|
||||
text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return DeepseekV3ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiK25Config,
|
||||
|
||||
@@ -120,15 +120,28 @@ def _language_model_config(config: KimiVLConfig):
|
||||
return text_config
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
return None if hf_config.encoder_only else _language_model_config(hf_config)
|
||||
|
||||
|
||||
class KimiVLForConditionalGeneration(nn.Module):
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
if hf_config.encoder_only:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
return DeepseekV2ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
_language_model_config(hf_config), quant_config
|
||||
text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return DeepseekV2ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiVLConfig,
|
||||
|
||||
@@ -67,7 +67,7 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.topk import TopK
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.mxfp8_input import (
|
||||
@@ -384,9 +384,7 @@ class MiniMaxM3MoE(nn.Module):
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.alt_stream = alt_stream
|
||||
self.n_shared_experts = getattr(config, "n_shared_experts", None)
|
||||
self.num_fused_shared_experts = (
|
||||
0 if is_shared_experts_fusion_disabled() else config.n_shared_experts
|
||||
)
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
|
||||
if self.tp_size > config.num_local_experts:
|
||||
raise ValueError(
|
||||
@@ -424,12 +422,11 @@ class MiniMaxM3MoE(nn.Module):
|
||||
gate_up_interleaved=False,
|
||||
)
|
||||
self.topk = TopK(
|
||||
top_k=config.num_experts_per_tok + self.num_fused_shared_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
renormalize=True,
|
||||
layer_id=layer_id,
|
||||
scoring_func=config.scoring_func,
|
||||
correction_bias=self.e_score_correction_bias,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=True,
|
||||
)
|
||||
@@ -1763,8 +1760,10 @@ class MiniMaxM3SparseForCausalLM(nn.Module):
|
||||
self.quant_config = quant_config
|
||||
self.pp_group = get_parallel().pp_group
|
||||
|
||||
self.num_fused_shared_experts = 0
|
||||
self.determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
if self.num_fused_shared_experts:
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
|
||||
self.model = MiniMaxM3Model(
|
||||
config, quant_config, prefix=add_prefix("model", prefix)
|
||||
@@ -1811,15 +1810,13 @@ class MiniMaxM3SparseForCausalLM(nn.Module):
|
||||
return "Shared experts fusion is not supported when Deepep MoE backend is enabled."
|
||||
return None
|
||||
|
||||
def determine_num_fused_shared_experts(self):
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
if is_shared_experts_fusion_disabled():
|
||||
return
|
||||
self.num_fused_shared_experts = self.config.n_shared_experts
|
||||
assert self.num_fused_shared_experts == 1, (
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Shared experts fused as routed slots when fusion is on."""
|
||||
assert hf_config.n_shared_experts == 1, (
|
||||
"Only 1 fused shared expert is supported for MiniMax-M3"
|
||||
)
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
return hf_config.n_shared_experts
|
||||
|
||||
def set_eagle3_layers_to_capture(self, layer_ids: Optional[list[int]] = None):
|
||||
if not self.pp_group.is_last_rank:
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch.nn as nn
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
get_moe_a2a_backend,
|
||||
is_shared_experts_fusion_disabled,
|
||||
get_num_fused_shared_experts,
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
@@ -88,8 +88,10 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
|
||||
|
||||
self.use_data_parallel = get_mm().mm_enable_dp_encoder
|
||||
|
||||
self.num_fused_shared_experts = 0
|
||||
self._determine_num_fused_shared_experts()
|
||||
# The weight loader reads this after the build.
|
||||
self.num_fused_shared_experts = get_num_fused_shared_experts()
|
||||
if self.num_fused_shared_experts:
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
|
||||
vision_config_raw = config.vision_config
|
||||
assert vision_config_raw is not None, "vision_config is required"
|
||||
@@ -177,15 +179,12 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
|
||||
)
|
||||
return None
|
||||
|
||||
def _determine_num_fused_shared_experts(self) -> None:
|
||||
# The decision was installed by the loader; this only reads it.
|
||||
if is_shared_experts_fusion_disabled():
|
||||
return
|
||||
self.num_fused_shared_experts = self.config.text_config.n_shared_experts
|
||||
assert self.num_fused_shared_experts == 1, (
|
||||
"Only 1 fused shared expert is supported"
|
||||
)
|
||||
log_info_on_rank0(logger, "Shared experts fusion optimization enabled.")
|
||||
@classmethod
|
||||
def num_fusable_shared_experts(cls, hf_config) -> int:
|
||||
"""Return the model's fusable shared-expert count."""
|
||||
num_shared_experts = hf_config.text_config.n_shared_experts
|
||||
assert num_shared_experts == 1, "Only 1 fused shared expert is supported"
|
||||
return num_shared_experts
|
||||
|
||||
@classmethod
|
||||
def get_model_config_for_expert_location(cls, config):
|
||||
|
||||
@@ -68,19 +68,33 @@ class VisionEncoderArgs:
|
||||
mm_projector_id: str = ""
|
||||
|
||||
|
||||
def _fused_moe_text_config(hf_config):
|
||||
"""Return the nested fusable MoE config, or None for other branches."""
|
||||
text_config = hf_config.text_config
|
||||
if getattr(text_config, "model_type", "") != "deepseek_v3":
|
||||
return None
|
||||
return text_config
|
||||
|
||||
|
||||
class PixtralForConditionalGeneration(nn.Module):
|
||||
merge_by_field_config = True
|
||||
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
text_config = hf_config.text_config
|
||||
if getattr(text_config, "model_type", "") != "deepseek_v3":
|
||||
# The GQA text config builds the dense Mistral backbone.
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return None
|
||||
return MistralLarge3ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
text_config, quant_config
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def num_fusable_shared_experts(hf_config) -> int:
|
||||
text_config = _fused_moe_text_config(hf_config)
|
||||
if text_config is None:
|
||||
return 0
|
||||
return MistralLarge3ForCausalLM.num_fusable_shared_experts(text_config)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
|
||||
@@ -557,19 +557,7 @@ class MoeFlags(_FlagGroupBase):
|
||||
tbo_token_distribution_threshold: float | None = None
|
||||
disable_fp4_allgather: bool | None = None
|
||||
quantization: str | None = None
|
||||
# The shared-experts-fusion decision, per runner — the runner_backend /
|
||||
# speculative_runner_backend shape. Both leaves are seeded from the config
|
||||
# intent by ``initialize_moe_config``; each MoE model's gate
|
||||
# (determine_num_fused_shared_experts) refines the ACTIVE leaf, both ways,
|
||||
# before its layers build and read it. ``speculative_moe_backend_context``
|
||||
# brackets a draft's build: on exit the draft's effective decision is
|
||||
# persisted onto the speculative leaf (inspectable afterwards) and the
|
||||
# target's ACTIVE value returns.
|
||||
disable_shared_experts_fusion: bool | None = None
|
||||
speculative_disable_shared_experts_fusion: bool | None = None
|
||||
# Lifecycle marker (the capture.disable_dispose_tensor family): set while
|
||||
# speculative_moe_backend_context is active, so a draft gate's write also
|
||||
# lands on the speculative leaf.
|
||||
# True while draft construction/backend scopes are active.
|
||||
in_speculative_scope: bool = False
|
||||
# Draft construction/execution uses a separate one-sided A2A workspace from
|
||||
# the target model's concurrently live CUDA graphs.
|
||||
|
||||
@@ -300,7 +300,8 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self._metadata_glue = None
|
||||
|
||||
def _replay_graph(self, shape_key, forward_batch):
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
with self.model_context():
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Helpers
|
||||
|
||||
@@ -307,7 +307,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
)
|
||||
|
||||
def _replay_graph(self, shape_key, forward_batch):
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
with self.model_context():
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
|
||||
def _cache_loc_dtype(self):
|
||||
return torch.int64
|
||||
|
||||
@@ -203,7 +203,8 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
return ShapeKey(size=bs)
|
||||
|
||||
def _replay_graph(self, shape_key, forward_batch):
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
with self.model_context():
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
|
||||
def can_run_graph(self, forward_batch: ForwardBatch):
|
||||
# Uniform-width replay invariant: the batch's actual per-request width
|
||||
|
||||
@@ -227,7 +227,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
)
|
||||
|
||||
def _replay_graph(self, shape_key, forward_batch):
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
with self.model_context():
|
||||
return self.backend.replay(shape_key, forward_batch)
|
||||
|
||||
def _make_graph_key(self, bs, stream_idx=None, variant_label=None):
|
||||
return ShapeKey(size=bs)
|
||||
@@ -412,11 +413,12 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
output_cache_loc_backup = forward_batch.out_cache_loc
|
||||
hidden_states_backup = forward_batch.spec_info.hidden_states
|
||||
|
||||
ret = self.model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
)
|
||||
with self.model_context():
|
||||
ret = self.model_runner.model.forward(
|
||||
forward_batch.input_ids,
|
||||
forward_batch.positions,
|
||||
forward_batch,
|
||||
)
|
||||
|
||||
if self.eagle_worker.chain_mtp_hidden_states and ret.hidden_states is not None:
|
||||
buffers.hidden_states[:num_tokens].copy_(ret.hidden_states[:num_tokens])
|
||||
@@ -1083,7 +1085,8 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
|
||||
with device_timer_ctx(
|
||||
first.model_runner.device_timer, "eagle_draft_extend"
|
||||
):
|
||||
outs = first.backend.replay(shape_key, self._replay_spec_info)
|
||||
with first.model_context():
|
||||
outs = first.backend.replay(shape_key, self._replay_spec_info)
|
||||
raw_bs = self.raw_bs
|
||||
self._cached = {}
|
||||
non_null = [r for r in self.runners if r is not None]
|
||||
|
||||
@@ -49,9 +49,9 @@ class RoutedExpertsCapturer(BaseTopkCapturer):
|
||||
) -> Optional["RoutedExpertsCapturer"]:
|
||||
if not get_exec().features.enable_return_routed_experts:
|
||||
return None
|
||||
# The model's own attribute is the baked decision (0 when its gate
|
||||
# disabled fusion); the ACTIVE flag can be holding another runner's.
|
||||
num_fused_shared_experts = getattr(model, "num_fused_shared_experts", 0)
|
||||
# The loader attaches this to every entry model, including wrappers.
|
||||
# Size from the owner, independent of which runner is currently active.
|
||||
num_fused_shared_experts = model.shared_experts_config.num_fused_shared_experts
|
||||
return RoutedExpertsCapturer(
|
||||
model_config,
|
||||
num_tokens=num_tokens,
|
||||
|
||||
+2
@@ -8,6 +8,7 @@ from unittest.mock import patch
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -672,6 +673,7 @@ class _EagleDraftExtendForward(nn.Module):
|
||||
device: str,
|
||||
):
|
||||
super().__init__()
|
||||
self.shared_experts_config = SharedExpertsConfig()
|
||||
self.module = module
|
||||
self.token_embed = nn.Embedding(
|
||||
vocab_size, hidden_size, dtype=dtype, device=device
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import patch
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.configs.shared_experts import SharedExpertsConfig
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
@@ -153,6 +154,10 @@ class _DummyTpGroup:
|
||||
|
||||
|
||||
class _TinyDraftModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.shared_experts_config = SharedExpertsConfig()
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
raise RuntimeError("EAGLEDraftCudaGraphRunner should call worker.draft_forward")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user