[Fix] deepseek-v4: Keep the draft MoE out of the target's expert placement (#45299)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
This commit is contained in:
@@ -359,6 +359,7 @@ class FusedMoE(torch.nn.Module):
|
||||
gate_up_interleaved: bool = True,
|
||||
enable_qwen35_fp8_deferred_finalize: bool = False,
|
||||
quant_method: Optional[FusedMoEMethodBase] = None,
|
||||
uses_expert_location: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
if params_dtype is None:
|
||||
@@ -367,6 +368,7 @@ class FusedMoE(torch.nn.Module):
|
||||
self.params_dtype = params_dtype
|
||||
self.layer_name = prefix
|
||||
self.layer_id = layer_id
|
||||
self.uses_expert_location = uses_expert_location
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.num_experts = num_experts
|
||||
@@ -1058,7 +1060,9 @@ class FusedMoE(torch.nn.Module):
|
||||
param.data[:, :dim1, :dim2].copy_(loaded_weight)
|
||||
return
|
||||
|
||||
global_expert_location_metadata = get_global_expert_location_metadata()
|
||||
global_expert_location_metadata = (
|
||||
get_global_expert_location_metadata() if self.uses_expert_location else None
|
||||
)
|
||||
if global_expert_location_metadata is None:
|
||||
if not getattr(param, "_sglang_require_global_experts", False):
|
||||
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
|
||||
|
||||
@@ -268,7 +268,11 @@ def _run_mega_routed(
|
||||
if forward_batch is not None
|
||||
else None
|
||||
)
|
||||
dispatch_info = ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
dispatch_info = (
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
)
|
||||
if (
|
||||
not moe.is_hash
|
||||
and getattr(moe.gate, "e_score_correction_bias_vl", None) is not None
|
||||
|
||||
@@ -312,8 +312,10 @@ def _run_mega_routed(
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=moe.layer_id
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=moe.layer_id)
|
||||
if moe.experts.uses_expert_location
|
||||
else None
|
||||
),
|
||||
**topk_kwargs,
|
||||
)
|
||||
|
||||
@@ -237,7 +237,7 @@ def _local_expert_ids(layer, expert_id: int) -> List[int]:
|
||||
metadata = get_global_expert_location_metadata()
|
||||
physical = (
|
||||
[expert_id]
|
||||
if metadata is None
|
||||
if metadata is None or not layer.uses_expert_location
|
||||
else metadata.logical_to_all_physical(layer.layer_id, expert_id)
|
||||
)
|
||||
local = (layer._map_global_expert_id_to_local_expert_id(p) for p in physical)
|
||||
|
||||
@@ -746,8 +746,12 @@ class DeepseekV2MoE(nn.Module):
|
||||
# with fused_shared_experts
|
||||
fused_shared_experts_scaling_factor = 1.0 / float(self.moe_ep_size)
|
||||
|
||||
uses_expert_location = not (is_nextn and is_deepseek_v4)
|
||||
num_redundant_experts = (
|
||||
get_exec().moe.ep_num_redundant_experts if uses_expert_location else 0
|
||||
)
|
||||
self.experts = get_moe_impl_class(quant_config)(
|
||||
num_experts=num_experts_for_moe + get_exec().moe.ep_num_redundant_experts,
|
||||
num_experts=num_experts_for_moe + num_redundant_experts,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
top_k=top_k_for_moe,
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -760,6 +764,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
),
|
||||
swiglu_limit=getattr(config, "swiglu_limit", None),
|
||||
prefix=add_prefix("experts", prefix),
|
||||
uses_expert_location=uses_expert_location,
|
||||
)
|
||||
|
||||
if self.is_hash and not (is_nextn and is_deepseek_v4):
|
||||
|
||||
@@ -990,6 +990,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
loaded_params = set()
|
||||
|
||||
weights = _dequant_fp8_wo_a_streaming(weights)
|
||||
n_routed_experts = _dspark_stage_config(self.config).n_routed_experts
|
||||
|
||||
stacked_params_mapping = DEEPSEEK_V4_STACKED_PARAMS_MAPPING
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
@@ -998,7 +999,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=(self.config.n_routed_experts + self.num_fused_shared_experts),
|
||||
num_experts=(n_routed_experts + self.num_fused_shared_experts),
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
@@ -1012,7 +1013,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
|
||||
if self.num_fused_shared_experts > 0 and ".mlp.shared_experts." in mapped:
|
||||
mapped = mapped.replace(
|
||||
".mlp.shared_experts.",
|
||||
f".mlp.experts.{self.config.n_routed_experts}.",
|
||||
f".mlp.experts.{n_routed_experts}.",
|
||||
)
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in mapped:
|
||||
|
||||
Reference in New Issue
Block a user