[CP] Support DP x TP x CP for attention with interleave CP strategy (#42038)

This commit is contained in:
Baizhou Zhang
2026-10-03 20:57:36 -07:00
committed by GitHub
parent f24db66729
commit fccc61c1c5
13 changed files with 318 additions and 67 deletions
@@ -91,30 +91,30 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
)
overrides["attn_dp_size"] = attn_dp_size
overrides["dp_size"] = 1
overrides["moe_dense_tp_size"] = 1
if cfg.cp_strategy == "zigzag":
overrides["moe_dense_tp_size"] = 1
overrides["moe_a2a_backend"] = "deepep"
overrides["ep_size"] = cfg.tp_size
logger.warning(
"zigzag DSA CP requires moe_dense_tp_size=1, "
"moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1."
)
else:
assert attn_dp_size == 1, (
"interleave DSA CP does not support DP attention."
)
assert cfg.tp_size <= 8, (
"Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues."
)
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
# The DSA / MLA CP gather and reduce-scatter
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
attn_cp_size = cfg.tp_size // attn_dp_size
# Interleave can shard attention heads within each CP rank.
# Keep an explicit CP width; default to attention TP1 as before.
# Zigzag still requires attention TP1.
attn_cp_size = (
cfg.attn_cp_size
if cfg.cp_strategy == "interleave" and cfg.attn_cp_size > 1
else cfg.tp_size // attn_dp_size
)
overrides["attn_cp_size"] = attn_cp_size
logger.warning(
"Enabled DSA context parallel: "
f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, "
f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, "
f"moe_dense_tp_size={overrides.get('moe_dense_tp_size', cfg.moe_dense_tp_size)}, "
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
f"attn_cp_size={attn_cp_size}, "
f"kv_cache_dtype={cfg.kv_cache_dtype}, "
+14 -11
View File
@@ -45,23 +45,26 @@ from sglang.srt.layers.cp.base import (
from sglang.srt.layers.cp.padding import pad_local_rows
from sglang.srt.layers.dp_attention import (
attn_cp_all_gather_into_tensor,
get_local_dp_buffer,
is_allocation_symmetric,
)
from sglang.srt.runtime_context import get_parallel
def attn_cp_interleave_gather(hidden_states: torch.Tensor):
"""Gather equal padded interleave shards in rank order, not token order."""
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
hidden_states, local_hidden_states = (
get_local_dp_buffer(get_parallel().attn_cp_group),
hidden_states,
)
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
return hidden_states
"""Gather equal padded interleave shards in rank order, not token order.
Size from the actual shard: the DP scratch length can already describe a
shard when dense FFNs run over TP, and is not a CP collective's output size.
"""
parallel = get_parallel()
with use_symmetric_memory(
parallel.attn_cp_group, disabled=not is_allocation_symmetric()
):
gathered = hidden_states.new_empty(
(hidden_states.shape[0] * parallel.attn_cp_size, *hidden_states.shape[1:])
)
attn_cp_all_gather_into_tensor(gathered, hidden_states.contiguous())
return gathered
@dataclass
+26 -3
View File
@@ -1069,6 +1069,15 @@ def can_use_dp_reduce_scatter() -> bool:
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
_note_dp_gather_in_prefill_graph()
parallel = get_parallel()
if parallel.attn_cp_size > 1 and input.shape[0] % parallel.tp_size:
# MAX_LEN padding aligns each DP slot to attention TP, not CP x TP.
# Small decode batches cannot be equally split over the full TP group.
reduced = parallel.tp_group.all_reduce(input)
output.copy_(
reduced.narrow(0, parallel.attn_dp_rank * output.shape[0], output.shape[0])
)
return
if is_dp_gatherv_active():
# Variable-length combine matching all_gatherv dispatch: scatter the
# global (sum_len) tensor back to per-rank token counts. Fall through to
@@ -1084,9 +1093,23 @@ def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
get_parallel().tp_rank
]
get_parallel().tp_group.reduce_scatter_tensor(scattered_local_tokens, input)
get_parallel().attn_tp_group.all_gather_into_tensor(
output, scattered_local_tokens
)
if parallel.attn_cp_size > 1:
# TP reduce-scatter splits each DP slot over CP x attention TP.
# Restore both axes: decode (and warmup) holds the full DP slot on
# every CP rank, even though prefill uses context-sharded rows.
cp_local_tokens = scattered_local_tokens
if parallel.attn_tp_size > 1:
cp_local_tokens = output.new_empty(
(output.shape[0] // parallel.attn_cp_size, *output.shape[1:])
)
parallel.attn_tp_group.all_gather_into_tensor(
cp_local_tokens, scattered_local_tokens
)
parallel.attn_cp_group.all_gather_into_tensor(output, cp_local_tokens)
else:
parallel.attn_tp_group.all_gather_into_tensor(
output, scattered_local_tokens
)
# ---------------------------------------------------------------------------
@@ -639,12 +639,15 @@ def _select_exit_move(
if not produced.may_reduce_scatter:
# A complete output: this rank's block of it, nothing summed.
return ExitMove(cp_moves.take_back)
# The FFN leaves its sum: only a take-back that sums over the same
# ranks completes it.
if cp_moves.reduce_scatter is None or not _same_ranks(
_sum_group(produced.group), cp_moves.reduce_scatter_group()
):
if cp_moves.reduce_scatter is None:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
# Only a take-back over the FFN's sum group can complete its reduction.
# With attention TP > 1, the full TP sum spans more ranks than CP:
# let the exit complete that sum, then take this rank's CP rows.
if not _same_ranks(_sum_group(produced.group), cp_moves.reduce_scatter_group()):
if produced.always_partial:
raise NotImplementedError(f"{produced=} {residual=} {to=}")
return ExitMove(cp_moves.take_back)
return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True)
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
# This rank's CP shard, from where the DP gather put it.
@@ -134,9 +134,6 @@ def attn_tp_reduce_scatter(
def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor):
"""Sum rank-major output onto each rank's equal, padded interleave shard."""
attn_dp_size = get_parallel().attn_dp_size
attn_tp_size = get_parallel().attn_tp_size
assert attn_dp_size == 1 and attn_tp_size == 1
cp_size = get_parallel().attn_cp_size
cp_rank = get_parallel().attn_cp_rank
input_hidden_states = hidden_states