[Fix] Shard Kimi K3's shared experts over the TP group they sum over (#42895)

This commit is contained in:
Cheng Wan
2026-10-09 01:26:15 -07:00
committed by GitHub
parent 45ad463c87
commit 7203747cc8
2 changed files with 64 additions and 52 deletions
+4
View File
@@ -618,6 +618,10 @@ class KimiK3MoE(nn.Module):
shared_experts_parallel_group = (
"shared_experts_tp" if requested_shared_tp is not None else "attn_tp"
)
else:
# The full TP group, which _reduce_shared sums over, whatever the
# dense MLP's sharding.
shared_experts_parallel_group = "tp"
if self.num_shared_experts is not None and self.num_shared_experts > 0:
shared_intermediate_size = moe_intermediate_size * self.num_shared_experts
if shared_tp is not None and shared_intermediate_size % shared_tp != 0: