[CP] Support DP x TP x CP for attention with interleave CP strategy (#42038)
This commit is contained in:
@@ -91,30 +91,30 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
overrides["attn_dp_size"] = attn_dp_size
|
||||
overrides["dp_size"] = 1
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
if cfg.cp_strategy == "zigzag":
|
||||
overrides["moe_dense_tp_size"] = 1
|
||||
overrides["moe_a2a_backend"] = "deepep"
|
||||
overrides["ep_size"] = cfg.tp_size
|
||||
logger.warning(
|
||||
"zigzag DSA CP requires moe_dense_tp_size=1, "
|
||||
"moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1."
|
||||
)
|
||||
else:
|
||||
assert attn_dp_size == 1, (
|
||||
"interleave DSA CP does not support DP attention."
|
||||
)
|
||||
assert cfg.tp_size <= 8, (
|
||||
"Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues."
|
||||
)
|
||||
# Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP.
|
||||
# The DSA / MLA CP gather and reduce-scatter
|
||||
# (the dsa_cp_* helpers in adapters/context_parallel.py) assume it.
|
||||
attn_cp_size = cfg.tp_size // attn_dp_size
|
||||
# Interleave can shard attention heads within each CP rank.
|
||||
# Keep an explicit CP width; default to attention TP1 as before.
|
||||
# Zigzag still requires attention TP1.
|
||||
attn_cp_size = (
|
||||
cfg.attn_cp_size
|
||||
if cfg.cp_strategy == "interleave" and cfg.attn_cp_size > 1
|
||||
else cfg.tp_size // attn_dp_size
|
||||
)
|
||||
overrides["attn_cp_size"] = attn_cp_size
|
||||
logger.warning(
|
||||
"Enabled DSA context parallel: "
|
||||
f"strategy={cfg.cp_strategy}, attn_dp_size={attn_dp_size}, "
|
||||
f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, "
|
||||
f"moe_dense_tp_size={overrides.get('moe_dense_tp_size', cfg.moe_dense_tp_size)}, "
|
||||
f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, "
|
||||
f"attn_cp_size={attn_cp_size}, "
|
||||
f"kv_cache_dtype={cfg.kv_cache_dtype}, "
|
||||
|
||||
@@ -45,23 +45,26 @@ from sglang.srt.layers.cp.base import (
|
||||
from sglang.srt.layers.cp.padding import pad_local_rows
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_all_gather_into_tensor,
|
||||
get_local_dp_buffer,
|
||||
is_allocation_symmetric,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
def attn_cp_interleave_gather(hidden_states: torch.Tensor):
|
||||
"""Gather equal padded interleave shards in rank order, not token order."""
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
hidden_states, local_hidden_states = (
|
||||
get_local_dp_buffer(get_parallel().attn_cp_group),
|
||||
hidden_states,
|
||||
)
|
||||
attn_cp_all_gather_into_tensor(hidden_states, local_hidden_states)
|
||||
return hidden_states
|
||||
"""Gather equal padded interleave shards in rank order, not token order.
|
||||
|
||||
Size from the actual shard: the DP scratch length can already describe a
|
||||
shard when dense FFNs run over TP, and is not a CP collective's output size.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
with use_symmetric_memory(
|
||||
parallel.attn_cp_group, disabled=not is_allocation_symmetric()
|
||||
):
|
||||
gathered = hidden_states.new_empty(
|
||||
(hidden_states.shape[0] * parallel.attn_cp_size, *hidden_states.shape[1:])
|
||||
)
|
||||
attn_cp_all_gather_into_tensor(gathered, hidden_states.contiguous())
|
||||
return gathered
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1069,6 +1069,15 @@ def can_use_dp_reduce_scatter() -> bool:
|
||||
|
||||
def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
|
||||
_note_dp_gather_in_prefill_graph()
|
||||
parallel = get_parallel()
|
||||
if parallel.attn_cp_size > 1 and input.shape[0] % parallel.tp_size:
|
||||
# MAX_LEN padding aligns each DP slot to attention TP, not CP x TP.
|
||||
# Small decode batches cannot be equally split over the full TP group.
|
||||
reduced = parallel.tp_group.all_reduce(input)
|
||||
output.copy_(
|
||||
reduced.narrow(0, parallel.attn_dp_rank * output.shape[0], output.shape[0])
|
||||
)
|
||||
return
|
||||
if is_dp_gatherv_active():
|
||||
# Variable-length combine matching all_gatherv dispatch: scatter the
|
||||
# global (sum_len) tensor back to per-rank token counts. Fall through to
|
||||
@@ -1084,9 +1093,23 @@ def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor):
|
||||
get_parallel().tp_rank
|
||||
]
|
||||
get_parallel().tp_group.reduce_scatter_tensor(scattered_local_tokens, input)
|
||||
get_parallel().attn_tp_group.all_gather_into_tensor(
|
||||
output, scattered_local_tokens
|
||||
)
|
||||
if parallel.attn_cp_size > 1:
|
||||
# TP reduce-scatter splits each DP slot over CP x attention TP.
|
||||
# Restore both axes: decode (and warmup) holds the full DP slot on
|
||||
# every CP rank, even though prefill uses context-sharded rows.
|
||||
cp_local_tokens = scattered_local_tokens
|
||||
if parallel.attn_tp_size > 1:
|
||||
cp_local_tokens = output.new_empty(
|
||||
(output.shape[0] // parallel.attn_cp_size, *output.shape[1:])
|
||||
)
|
||||
parallel.attn_tp_group.all_gather_into_tensor(
|
||||
cp_local_tokens, scattered_local_tokens
|
||||
)
|
||||
parallel.attn_cp_group.all_gather_into_tensor(output, cp_local_tokens)
|
||||
else:
|
||||
parallel.attn_tp_group.all_gather_into_tensor(
|
||||
output, scattered_local_tokens
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -639,12 +639,15 @@ def _select_exit_move(
|
||||
if not produced.may_reduce_scatter:
|
||||
# A complete output: this rank's block of it, nothing summed.
|
||||
return ExitMove(cp_moves.take_back)
|
||||
# The FFN leaves its sum: only a take-back that sums over the same
|
||||
# ranks completes it.
|
||||
if cp_moves.reduce_scatter is None or not _same_ranks(
|
||||
_sum_group(produced.group), cp_moves.reduce_scatter_group()
|
||||
):
|
||||
if cp_moves.reduce_scatter is None:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
# Only a take-back over the FFN's sum group can complete its reduction.
|
||||
# With attention TP > 1, the full TP sum spans more ranks than CP:
|
||||
# let the exit complete that sum, then take this rank's CP rows.
|
||||
if not _same_ranks(_sum_group(produced.group), cp_moves.reduce_scatter_group()):
|
||||
if produced.always_partial:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {to=}")
|
||||
return ExitMove(cp_moves.take_back)
|
||||
return ExitMove(cp_moves.reduce_scatter, output_move_completes_sum=True)
|
||||
if returned == {TokenAxis.ATTN_DP, TokenAxis.ATTN_CP}:
|
||||
# This rank's CP shard, from where the DP gather put it.
|
||||
|
||||
@@ -134,9 +134,6 @@ def attn_tp_reduce_scatter(
|
||||
|
||||
def attn_cp_interleave_reduce_scatter(hidden_states: torch.Tensor):
|
||||
"""Sum rank-major output onto each rank's equal, padded interleave shard."""
|
||||
attn_dp_size = get_parallel().attn_dp_size
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
assert attn_dp_size == 1 and attn_tp_size == 1
|
||||
cp_size = get_parallel().attn_cp_size
|
||||
cp_rank = get_parallel().attn_cp_rank
|
||||
input_hidden_states = hidden_states
|
||||
|
||||
Reference in New Issue
Block a user