fix(qwen3.5): restrict MoE weights to local PP layers (#32022)
Co-authored-by: chunjiang.hc <chunjiang.hc@alibaba-inc.com> Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com>
This commit is contained in:
co-authored by
chunjiang.hc
Yihao Wang
parent
f73d6f2789
commit
e4f7f7b380
@@ -2288,9 +2288,9 @@ class Qwen3_5MoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
||||
|
||||
self._routed_experts_weights_of_layer = LazyValue(
|
||||
lambda: {
|
||||
layer_id: layer.mlp.get_moe_weights()
|
||||
for layer_id, layer in enumerate(self.model.layers)
|
||||
if isinstance(layer.mlp, Qwen2MoeSparseMoeBlock)
|
||||
layer_id: self.model.layers[layer_id].mlp.get_moe_weights()
|
||||
for layer_id in range(self.model.start_layer, self.model.end_layer)
|
||||
if isinstance(self.model.layers[layer_id].mlp, Qwen2MoeSparseMoeBlock)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user