[Fix] Run MoE layers under attention DP and GQA prefill CP on the declared DP × CP gather (#41432)

This commit is contained in:
Cheng Wan
2026-09-27 16:52:10 -07:00
committed by GitHub
parent f0ecf15c84
commit 31329c222c
3 changed files with 24 additions and 7 deletions
+4 -2
View File
@@ -1020,12 +1020,14 @@ class LayerCommunicator:
if not (
(parallel.attn_cp_size == 1 or _cp_on_declarations())
# MoE layers under attention DP and GQA prefill CP keep the
# scatter-mode steps.
# A MoE dispatched per DP shard under attention DP and GQA prefill
# CP keeps the scatter-mode steps: it would dispatch each rank's CP
# shard of the tokens, which is not covered.
and not (
parallel.attn_cp_size > 1
and parallel.attn_dp_size > 1
and modes.is_layer_sparse
and moe_on_local_rows
and not _gathers_over_attention_cp()
)
and (modes.is_first_layer or modes.is_previous_layer_sparse is not None)