[Qwen3.8 CP 4/4] Prefill context parallel E2E support of Qwen3.8 (#39724)
This commit is contained in:
@@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
|
||||
# Architectures whose model code implements CP-TP group sharing: the CP group
|
||||
@@ -17,7 +18,10 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha
|
||||
# MoE and linear attention keep their TP partition (the linear attention
|
||||
# gathers the sequence where its recurrence needs it). Model ports register
|
||||
# here; every other model keeps the ordinary prefill CP path.
|
||||
_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"}
|
||||
_SUPPORTED_MODELS: set[str] = {
|
||||
"Glm5NextForConditionalGeneration",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}
|
||||
|
||||
|
||||
def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
@@ -44,25 +48,50 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
|
||||
# This hook runs before speculative defaults and NEXTN alias resolution.
|
||||
# Native MTP keeps the target's CP/TP placement; other draft architectures
|
||||
# have their own topology and model-boundary contracts.
|
||||
if cfg.speculative_algorithm is not None and (
|
||||
model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration"
|
||||
or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
|
||||
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
|
||||
or cfg.speculative_eagle_topk not in (None, 1)
|
||||
):
|
||||
raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.")
|
||||
# have their own topology and model-boundary contracts. Only GLM-5.3-Flash
|
||||
# has its MTP ported.
|
||||
if cfg.speculative_algorithm is not None:
|
||||
if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration":
|
||||
raise ValueError(
|
||||
"--speculative-algorithm is not supported with CP-TP group sharing."
|
||||
)
|
||||
if (
|
||||
cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
|
||||
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
|
||||
or cfg.speculative_eagle_topk not in (None, 1)
|
||||
):
|
||||
raise ValueError(
|
||||
"CP/TP group sharing only supports native MTP with top-k 1."
|
||||
)
|
||||
|
||||
unsupported = {
|
||||
"enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split,
|
||||
"dcp-size > 1": cfg.dcp_size > 1,
|
||||
"enable-mixed-chunk": cfg.enable_mixed_chunk,
|
||||
"moe-dp-size > 1": cfg.moe_dp_size > 1,
|
||||
"disaggregation-mode": cfg.disaggregation_mode != "null",
|
||||
}
|
||||
if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration":
|
||||
unsupported |= {
|
||||
# The MoE runs TP experts on rows gathered over the TP group.
|
||||
# Waterfill turns on DeepEP and EP after this hook.
|
||||
"ep-size > 1": view.ep_size > 1,
|
||||
"moe-a2a-backend": view.moe_a2a_backend != "none",
|
||||
"enable-waterfill": cfg.enable_waterfill,
|
||||
"pp-size > 1": cfg.pp_size > 1,
|
||||
# Prefill only: a decode node serves plain TP.
|
||||
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
|
||||
}
|
||||
else:
|
||||
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
|
||||
for flag, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
|
||||
# Decode sees this prefill as TP cp_size, while its sender stages by
|
||||
# attention TP 1, so the two would pick different staging layouts.
|
||||
if cfg.disaggregation_mode == "prefill" and envs.SGLANG_DISAGG_STAGING_BUFFER.get():
|
||||
raise ValueError(
|
||||
"SGLANG_DISAGG_STAGING_BUFFER is not supported with CP-TP group sharing."
|
||||
)
|
||||
|
||||
linear_config = mambaish_config(model)
|
||||
if getattr(linear_config, "linear_num_key_heads", None) is not None:
|
||||
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_serving,
|
||||
linear_attn_tp_size,
|
||||
max_prefill_buffer_tokens,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -320,6 +321,11 @@ class CommonKVManager(BaseKVManager):
|
||||
self.attn_tp_rank = parallel.attn_tp_rank
|
||||
self.attn_cp_size = parallel.attn_cp_size
|
||||
self.attn_cp_rank = parallel.attn_cp_rank
|
||||
# The TP width the transferred state is laid out over, which decode
|
||||
# pairs with: attention TP, or the TP group under CP-TP group sharing,
|
||||
# where the GDN heads are partitioned over it and QSA/PLE state is whole
|
||||
# on every rank. KV heads keep following attention TP.
|
||||
self.state_tp_size = linear_attn_tp_size()
|
||||
self.dcp_size = parallel.attn_dcp_size
|
||||
self.dcp_rank = parallel.attn_dcp_rank
|
||||
self.attn_dp_size = parallel.attn_dp_size
|
||||
@@ -339,7 +345,11 @@ class CommonKVManager(BaseKVManager):
|
||||
self.is_hybrid_mla_backend
|
||||
and disaggregation_mode == DisaggregationMode.DECODE
|
||||
)
|
||||
self.enable_all_cp_ranks_for_transfer = (
|
||||
# CP-TP group sharing: CP rank r holds every KV head, the CP-replicated
|
||||
# state and linear-attention head shard r. It transfers as TP rank r of
|
||||
# a CP-1 prefill (see register_to_bootstrap), not as a CP replica.
|
||||
cp_tp_group_sharing = parallel.enable_cp_tp_group_sharing
|
||||
self.enable_all_cp_ranks_for_transfer = not cp_tp_group_sharing and (
|
||||
envs.SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER.get()
|
||||
or cp_sharded_prefill
|
||||
or hybrid_decode_pulls_all_ranks
|
||||
@@ -389,6 +399,7 @@ class CommonKVManager(BaseKVManager):
|
||||
# participate in KV transfer; Otherwise only CP rank 0 sends.
|
||||
self.is_dummy_cp_rank = (
|
||||
not self.enable_all_cp_ranks_for_transfer
|
||||
and not cp_tp_group_sharing
|
||||
and self.attn_cp_size > 1
|
||||
and self.attn_cp_rank != 0
|
||||
)
|
||||
@@ -471,6 +482,7 @@ class CommonKVManager(BaseKVManager):
|
||||
self.attn_cp_size > 1
|
||||
and self.attn_cp_rank != 0
|
||||
and not get_parallel().enable_dsa_cache_layer_split
|
||||
and not get_parallel().enable_cp_tp_group_sharing
|
||||
)
|
||||
|
||||
def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool:
|
||||
@@ -1396,11 +1408,16 @@ class CommonKVManager(BaseKVManager):
|
||||
|
||||
bootstrap_na = NetworkAddress(host, self.bootstrap_port)
|
||||
url = f"{bootstrap_na.to_url()}/route"
|
||||
attn_tp = (self.attn_tp_size, self.attn_tp_rank)
|
||||
attn_cp = (self.attn_cp_size, self.attn_cp_rank)
|
||||
if get_parallel().enable_cp_tp_group_sharing:
|
||||
# Decode pairs its TP rank r with this CP rank r (see __init__).
|
||||
attn_tp, attn_cp = attn_cp, (1, 0)
|
||||
payload = {
|
||||
"attn_tp_size": self.attn_tp_size,
|
||||
"attn_tp_rank": self.attn_tp_rank,
|
||||
"attn_cp_size": self.attn_cp_size,
|
||||
"attn_cp_rank": self.attn_cp_rank,
|
||||
"attn_tp_size": attn_tp[0],
|
||||
"attn_tp_rank": attn_tp[1],
|
||||
"attn_cp_size": attn_cp[0],
|
||||
"attn_cp_rank": attn_cp[1],
|
||||
"attn_dp_size": self.attn_dp_size,
|
||||
"attn_dp_rank": self.attn_dp_rank,
|
||||
"pp_size": self.pp_size,
|
||||
|
||||
@@ -1959,7 +1959,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
)
|
||||
if (
|
||||
target_rank_registration_info is not None
|
||||
and self.attn_tp_size
|
||||
and self.state_tp_size
|
||||
!= target_rank_registration_info.dst_attn_tp_size
|
||||
):
|
||||
rc = (
|
||||
@@ -2291,7 +2291,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"""
|
||||
logger.warning_once(
|
||||
"Using Mamba state slice transfer for different runtime attention TP "
|
||||
f"sizes: prefill={self.attn_tp_size}, decode={dst_attn_tp_size}. "
|
||||
f"sizes: prefill={self.state_tp_size}, decode={dst_attn_tp_size}. "
|
||||
"Performance may be affected."
|
||||
)
|
||||
assert len(prefill_mamba_index) == 1, "Mamba should have single state index"
|
||||
@@ -2310,7 +2310,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_state_item_lens,
|
||||
)
|
||||
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size
|
||||
dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size
|
||||
|
||||
transfer_blocks = []
|
||||
@@ -2349,7 +2349,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_dim=src_dim,
|
||||
dst_dim=dst_dim,
|
||||
outer_count=outer_count,
|
||||
src_attn_tp_size=self.attn_tp_size,
|
||||
src_attn_tp_size=self.state_tp_size,
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
dst_tp_rank_in_group=dst_tp_rank_in_group,
|
||||
local_tp_rank_in_group=local_tp_rank_in_group,
|
||||
|
||||
@@ -1654,7 +1654,7 @@ class MoriKVManager(CommonKVManager):
|
||||
)
|
||||
|
||||
if st == "mamba":
|
||||
if peer_info.decode_tp_size != self.attn_tp_size and 0 in src_dims:
|
||||
if peer_info.decode_tp_size != self.state_tp_size and 0 in src_dims:
|
||||
raise RuntimeError(
|
||||
"Replicated Mamba PD state transfer currently requires "
|
||||
"matching prefill/decode attention TP sizes"
|
||||
@@ -1727,7 +1727,7 @@ class MoriKVManager(CommonKVManager):
|
||||
f"got src={src_state_indices.size}, dst={dst_state_indices.size}"
|
||||
)
|
||||
|
||||
tp_mismatch = peer_info.decode_tp_size != self.attn_tp_size
|
||||
tp_mismatch = peer_info.decode_tp_size != self.state_tp_size
|
||||
|
||||
# If dim info missing, silently degrade to whole-item copy (Mooncake compat)
|
||||
if tp_mismatch and (
|
||||
@@ -1738,7 +1738,7 @@ class MoriKVManager(CommonKVManager):
|
||||
if tp_mismatch:
|
||||
logger.warning_once(
|
||||
"Using Mamba state slice transfer for different TP sizes between prefill and decode. "
|
||||
f"Prefill attn_tp_size={self.attn_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. "
|
||||
f"Prefill attn_tp_size={self.state_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. "
|
||||
"Performance may be affected."
|
||||
)
|
||||
|
||||
@@ -1746,7 +1746,7 @@ class MoriKVManager(CommonKVManager):
|
||||
dst_idx = int(dst_state_indices[0])
|
||||
statuses: List[TransferStatus] = []
|
||||
|
||||
local_tp_rank = self.kv_args.engine_rank % self.attn_tp_size
|
||||
local_tp_rank = self.kv_args.engine_rank % self.state_tp_size
|
||||
dst_tp_rank = peer_info.decode_tp_rank % peer_info.decode_tp_size
|
||||
|
||||
pairs = build_transfer_entry_pairs(
|
||||
@@ -1792,7 +1792,7 @@ class MoriKVManager(CommonKVManager):
|
||||
src_dim=src_dim,
|
||||
dst_dim=dst_dim,
|
||||
outer_count=outer_count,
|
||||
src_attn_tp_size=self.attn_tp_size,
|
||||
src_attn_tp_size=self.state_tp_size,
|
||||
dst_attn_tp_size=peer_info.decode_tp_size,
|
||||
dst_tp_rank_in_group=dst_tp_rank,
|
||||
local_tp_rank_in_group=local_tp_rank,
|
||||
@@ -1853,7 +1853,13 @@ class MoriKVManager(CommonKVManager):
|
||||
"PP>1 needs peer state_layer_ids for global-layer descriptor "
|
||||
"pairing."
|
||||
)
|
||||
if peer_info.decode_tp_size != self.attn_tp_size:
|
||||
# QSA state is whole on every rank, so it pairs by state TP.
|
||||
src_tp_size = (
|
||||
self.state_tp_size
|
||||
if state_type in ("qsa_pending", "qsa_compressed")
|
||||
else self.attn_tp_size
|
||||
)
|
||||
if peer_info.decode_tp_size != src_tp_size:
|
||||
raise RuntimeError(
|
||||
f"PD disagg: heterogeneous TP not supported for {state_type} yet."
|
||||
)
|
||||
|
||||
@@ -2539,7 +2539,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"""
|
||||
logger.warning_once(
|
||||
"Using Mamba state slice transfer for different TP sizes. "
|
||||
f"Prefill attn_tp_size={self.attn_tp_size}, "
|
||||
f"Prefill attn_tp_size={self.state_tp_size}, "
|
||||
f"Decode attn_tp_size={decode_tp_size}."
|
||||
)
|
||||
assert len(prefill_state_indices) == 1, "Mamba should have single state index"
|
||||
@@ -2558,7 +2558,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_layer_ids=dst_layer_ids,
|
||||
)
|
||||
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size
|
||||
dst_tp_rank_in_group = decode_tp_rank % decode_tp_size
|
||||
|
||||
src_addrs = []
|
||||
@@ -2601,7 +2601,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_dim=src_dim,
|
||||
dst_dim=dst_dim,
|
||||
outer_count=outer_count,
|
||||
src_attn_tp_size=self.attn_tp_size,
|
||||
src_attn_tp_size=self.state_tp_size,
|
||||
dst_attn_tp_size=decode_tp_size,
|
||||
dst_tp_rank_in_group=dst_tp_rank_in_group,
|
||||
local_tp_rank_in_group=local_tp_rank_in_group,
|
||||
@@ -2761,7 +2761,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
comp_notif = f"{notif}_{i}"
|
||||
|
||||
if st == StateType.MAMBA:
|
||||
if self.attn_tp_size != decode_tp_size:
|
||||
if self.state_tp_size != decode_tp_size:
|
||||
if 0 in src_dims:
|
||||
raise RuntimeError(
|
||||
"Replicated Mamba PD state transfer currently requires "
|
||||
@@ -2856,7 +2856,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
StateType.SWA_RING,
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
):
|
||||
if not self.is_mla_backend and self.attn_tp_size != decode_tp_size:
|
||||
# QSA state is whole on every rank, so it pairs by state TP.
|
||||
src_tp_size = (
|
||||
self.state_tp_size
|
||||
if st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED)
|
||||
else self.attn_tp_size
|
||||
)
|
||||
if not self.is_mla_backend and src_tp_size != decode_tp_size:
|
||||
raise RuntimeError(
|
||||
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
|
||||
)
|
||||
|
||||
@@ -42,6 +42,8 @@ from sglang.srt.layers.layer_boundary.ops import (
|
||||
attn_cp_take_back_output,
|
||||
attn_tp_gather_input,
|
||||
attn_tp_slice_output,
|
||||
cp_gather_in_token_order,
|
||||
cp_take_back_in_token_order,
|
||||
dp_cp_take_back_output,
|
||||
keep_output,
|
||||
moe_cp_take_back_output,
|
||||
@@ -444,15 +446,31 @@ def _select_entry_step(
|
||||
# A head-parallel mixer consumes full context, but its residual and
|
||||
# readout coefficients remain on the CP shard. Complete the producer's
|
||||
# sum onto that shard before the (possibly nonlinear) update/read.
|
||||
if (
|
||||
residual != residual_to
|
||||
or cp_moves is None
|
||||
or owes not in (None, SumGroup.ATTN_CP)
|
||||
or (owes is not None and cp_moves.reduce_scatter is None)
|
||||
if residual != residual_to or cp_moves is None:
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
if produced.in_token_order:
|
||||
# The whole sequence in token order (a scan's): complete the sum on
|
||||
# every row, then take this rank's shard back. No model runs two
|
||||
# token-order stages back to back, so that edge is not bound.
|
||||
if need.in_token_order or owes not in (
|
||||
None,
|
||||
SumGroup.ATTN_CP,
|
||||
SumGroup.TP,
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
move = partial(cp_take_back_in_token_order, group=owes)
|
||||
elif owes not in (None, SumGroup.ATTN_CP) or (
|
||||
owes is not None and cp_moves.reduce_scatter is None
|
||||
):
|
||||
raise NotImplementedError(f"{produced=} {residual=} {need=}")
|
||||
else:
|
||||
move = cp_moves.reduce_scatter if owes is not None else cp_moves.take_back
|
||||
local = msgspec.structs.replace(
|
||||
produced, layout=residual_to, group=None, always_partial=False
|
||||
produced,
|
||||
layout=residual_to,
|
||||
group=None,
|
||||
always_partial=False,
|
||||
in_token_order=False,
|
||||
)
|
||||
inner = _select_entry_step(
|
||||
local,
|
||||
@@ -466,13 +484,7 @@ def _select_entry_step(
|
||||
enters_stack=enters_stack,
|
||||
)
|
||||
return inner._replace(
|
||||
step=partial(
|
||||
_move_before_read,
|
||||
move=cp_moves.reduce_scatter
|
||||
if owes is not None
|
||||
else cp_moves.take_back,
|
||||
read=inner.step,
|
||||
)
|
||||
step=partial(_move_before_read, move=move, read=inner.step)
|
||||
)
|
||||
if sliced:
|
||||
# Each attention-TP rank takes its own slice: the reduce-scatter
|
||||
@@ -549,6 +561,9 @@ def _select_entry_step(
|
||||
cp_moves=cp_moves,
|
||||
enters_stack=enters_stack,
|
||||
).step
|
||||
if need.in_token_order:
|
||||
# The stage scans the whole sequence: gather it in token order.
|
||||
return EntryStep(on_chunk, input_move=cp_gather_in_token_order)
|
||||
return EntryStep(partial(cp_moves.gather, gather=on_chunk))
|
||||
if gathered == {TokenAxis.ATTN_TP}:
|
||||
# A complete input on each rank's slice, gathered over attention TP
|
||||
|
||||
@@ -290,6 +290,8 @@ def _bind_stage(
|
||||
and TokenAxis.ATTN_CP
|
||||
in edge.produced.layout.sharded - edge.need.layout.sharded
|
||||
and not _cp_gathers_over_attn_cp()
|
||||
# CP-TP group sharing is the MoE-CP gather MHC has been run with.
|
||||
and not get_parallel().enable_cp_tp_group_sharing
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with a gather over attention CP"
|
||||
|
||||
@@ -71,6 +71,9 @@ class InputContract(msgspec.Struct, frozen=True):
|
||||
gathered_by_compute: Token axes compute can gather internally; the boundary
|
||||
may hand it input still sharded over those axes.
|
||||
read: Operation that derives input from the updated residual.
|
||||
in_token_order: Compute reads the whole sequence in the batch's token
|
||||
order (a scan over it), so a gather over attention CP restores
|
||||
that order instead of placing the shards side by side.
|
||||
"""
|
||||
|
||||
layout: Layout
|
||||
@@ -79,6 +82,7 @@ class InputContract(msgspec.Struct, frozen=True):
|
||||
gathered_by_compute: FrozenSet[TokenAxis] = frozenset()
|
||||
# How the consumer reads its input from the residual.
|
||||
read: ResidualReadout = NORM_READOUT
|
||||
in_token_order: bool = False
|
||||
|
||||
|
||||
class OutputContract(msgspec.Struct, frozen=True):
|
||||
@@ -99,6 +103,8 @@ class OutputContract(msgspec.Struct, frozen=True):
|
||||
update: Producer residual operation; None on an arrival contract that
|
||||
declares capabilities and obtains the actual update from the stream.
|
||||
transform: Optional operation on the contribution before residual update.
|
||||
in_token_order: The output covers the whole sequence in the batch's
|
||||
token order; the consumer takes this rank's CP shard of it back.
|
||||
|
||||
These are permissions, not a record of a particular output. The exit
|
||||
decision and ResidualStream record what that output actually owes.
|
||||
@@ -118,6 +124,7 @@ class OutputContract(msgspec.Struct, frozen=True):
|
||||
# description has no producer object; its edge declares capabilities only.
|
||||
update: Optional[ResidualUpdate] = PLAIN_ADD
|
||||
transform: Optional[OutputTransform] = None
|
||||
in_token_order: bool = False
|
||||
|
||||
|
||||
class StageContract(msgspec.Struct, frozen=True):
|
||||
|
||||
@@ -286,6 +286,8 @@ class StageDeclaration:
|
||||
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
|
||||
tp_group: Group partitioning attention heads. Full TP consumers require
|
||||
full token rows, including when TP shares the prefill CP group.
|
||||
in_token_order: Whether an attention reads the whole sequence in token
|
||||
order (a scan over it), e.g. linear attention under prefill CP.
|
||||
dense_tp_size: Dense FFN compute width: None uses the configured width,
|
||||
1 means local compute, the attention TP size means compute on the
|
||||
attention's rows over attention TP, and the full TP size means TP
|
||||
@@ -324,6 +326,7 @@ class StageDeclaration:
|
||||
reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED
|
||||
gathers_attn_tp_input: bool = False
|
||||
tp_group: SumGroup = SumGroup.ATTN_TP
|
||||
in_token_order: bool = False
|
||||
dense_tp_size: Optional[int] = None
|
||||
exit_rows: Optional[ExitRows] = None
|
||||
writes_at_handoff: bool = False
|
||||
@@ -419,6 +422,7 @@ def declare_attn(
|
||||
output_transform=None,
|
||||
tp_group=SumGroup.ATTN_TP,
|
||||
attn_tp_gather=None,
|
||||
in_token_order=False,
|
||||
):
|
||||
"""Declare attention or a mixer; construct its executable boundary later.
|
||||
|
||||
@@ -435,15 +439,20 @@ def declare_attn(
|
||||
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
|
||||
attn_tp_gather: Implementation of the gather over attention TP into
|
||||
this stage, tried before the boundary's own (see StageDeclaration).
|
||||
in_token_order: Compute scans the whole sequence in token order. On a
|
||||
CP extend its input is gathered into that order, and the next
|
||||
stage's input completes the sum and takes this rank's shard back.
|
||||
Requires ALWAYS_PARTIAL.
|
||||
|
||||
Returns:
|
||||
A StageDeclaration with no norm, tensors or execution plan.
|
||||
"""
|
||||
if (
|
||||
output_transform is not None
|
||||
and reduction is not ProducerReduction.ALWAYS_PARTIAL
|
||||
):
|
||||
raise ValueError("an attention output transform requires ALWAYS_PARTIAL")
|
||||
output_transform is not None or in_token_order
|
||||
) and reduction is not ProducerReduction.ALWAYS_PARTIAL:
|
||||
raise ValueError(
|
||||
"an attention output transform or token-order input requires ALWAYS_PARTIAL"
|
||||
)
|
||||
return StageDeclaration(
|
||||
StageKind.ATTENTION,
|
||||
read,
|
||||
@@ -453,6 +462,7 @@ def declare_attn(
|
||||
gathers_attn_tp_input=gathers_attn_tp_input,
|
||||
tp_group=tp_group,
|
||||
attn_tp_gather=attn_tp_gather,
|
||||
in_token_order=in_token_order,
|
||||
)
|
||||
|
||||
|
||||
@@ -504,7 +514,12 @@ def _reject_unsupported(stage):
|
||||
rows of one edge are made where that edge binds (``_bind_stage``)."""
|
||||
parallel = get_parallel()
|
||||
if stage.update.applied_at_exit:
|
||||
if stage.sparse and moe_gathers_over_moe_cp():
|
||||
# CP-TP group sharing is the MoE-CP gather MHC has been run with.
|
||||
if (
|
||||
stage.sparse
|
||||
and moe_gathers_over_moe_cp()
|
||||
and not parallel.enable_cp_tp_group_sharing
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with a MoE gathered over "
|
||||
"the MoE-CP group"
|
||||
@@ -614,8 +629,19 @@ def _resolve_stage(stage, variant, arrival, following=None):
|
||||
)
|
||||
elif stage.tp_group is not SumGroup.ATTN_TP:
|
||||
raise ValueError("attention heads must be partitioned over ATTN_TP or TP")
|
||||
# On a CP extend a token-order stage takes the whole sequence.
|
||||
in_token_order = stage.in_token_order and variant is BatchVariant.CONTEXT_PARALLEL
|
||||
if in_token_order:
|
||||
if get_parallel().attn_dp_size > 1:
|
||||
raise NotImplementedError("a token-order attention under attention DP")
|
||||
compute_rows = full
|
||||
declaration = StageContract(
|
||||
InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read),
|
||||
InputContract(
|
||||
compute_rows,
|
||||
gathered_by_compute=gathers,
|
||||
read=stage.read,
|
||||
in_token_order=in_token_order,
|
||||
),
|
||||
OutputContract(
|
||||
local if sp else compute_rows,
|
||||
group=group if owes else None,
|
||||
@@ -630,6 +656,7 @@ def _resolve_stage(stage, variant, arrival, following=None):
|
||||
and following.kind is StageKind.ATTENTION,
|
||||
update=stage.update,
|
||||
transform=stage.output_transform,
|
||||
in_token_order=in_token_order,
|
||||
),
|
||||
)
|
||||
# An attention moves no residual: it keeps the rows it arrives on, except
|
||||
|
||||
@@ -28,6 +28,10 @@ from sglang.srt.distributed import (
|
||||
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
|
||||
use_symmetric_memory,
|
||||
)
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_full_sequence_states,
|
||||
cp_shard_hidden_states,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_cp_reduce_scatter_tensor,
|
||||
attn_tp_all_gather_into_tensor,
|
||||
@@ -52,6 +56,7 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
TokenAxis,
|
||||
_cp_shard_token_rows,
|
||||
_sum_group,
|
||||
moe_cp_gathered_rows,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
|
||||
@@ -179,6 +184,28 @@ def moe_cp_gather(
|
||||
return output
|
||||
|
||||
|
||||
def cp_gather_in_token_order(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch
|
||||
) -> torch.Tensor:
|
||||
"""This rank's CP shard to the whole sequence in token order, without the
|
||||
shards' padding."""
|
||||
return cp_gather_full_sequence_states(hidden_states, forward_batch)
|
||||
|
||||
|
||||
def cp_take_back_in_token_order(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
group: Optional[SumGroup],
|
||||
):
|
||||
"""The whole sequence in token order back to this rank's CP shard,
|
||||
completing the sum it owes over ``group`` first."""
|
||||
if group is not None:
|
||||
hidden_states = _sum_group(group).all_reduce(hidden_states)
|
||||
return cp_shard_hidden_states(hidden_states, forward_batch), residual
|
||||
|
||||
|
||||
def attn_tp_all_reduce(
|
||||
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import math
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import Any, Iterable, Optional, Set, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
@@ -26,6 +27,12 @@ from sglang.srt.layers.attention.linear.utils import (
|
||||
select_verify_intermediate_state_indices,
|
||||
)
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.cp.utils import (
|
||||
cp_gather_after_forward,
|
||||
cp_shard_hidden_states,
|
||||
cp_split_before_forward,
|
||||
is_cp_active,
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_reduce,
|
||||
dp_gather_replicate,
|
||||
@@ -40,6 +47,7 @@ from sglang.srt.layers.hyperconnection import (
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
GatedResidualState,
|
||||
SumGroup,
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
@@ -89,7 +97,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
|
||||
make_ple_file_prefetcher,
|
||||
make_ple_file_rss_trimmer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel, linear_attn_parallel_group
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, logger
|
||||
|
||||
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
|
||||
@@ -1444,20 +1452,47 @@ def _has_ple(layer_id, config) -> bool:
|
||||
return (layer_id + 1) in config.ple_layer_ids
|
||||
|
||||
|
||||
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
|
||||
def _shards_rows_over_cp(forward_batch: ForwardBatch) -> bool:
|
||||
"""A CP extend under CP-TP group sharing: every layer runs on this rank's
|
||||
zigzag token rows, and the GDN and the PLE gather the sequence in global
|
||||
token order where they need it. The batch metadata stays global."""
|
||||
return (
|
||||
get_parallel().enable_cp_tp_group_sharing
|
||||
and forward_batch.attn_cp_metadata is not None
|
||||
and is_cp_active(forward_batch)
|
||||
)
|
||||
|
||||
|
||||
def _linear_attn_options():
|
||||
"""Linear attention scans the sequence in token order, and its output owes
|
||||
the sum over the group its heads are partitioned over."""
|
||||
return dict(
|
||||
tp_group=(
|
||||
SumGroup.TP if linear_attn_parallel_group() == "tp" else SumGroup.ATTN_TP
|
||||
),
|
||||
in_token_order=True,
|
||||
)
|
||||
|
||||
|
||||
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None, linear_attention=False):
|
||||
"""The attention and FFN stages a gated hyper-connection layer of either
|
||||
kind declares: the model's shared declaration function (see make_layers),
|
||||
which the layer declares itself with. ``residual`` is the layer's own
|
||||
reads and updates of the streams; without it, the stages declare what
|
||||
those do. A PLE layer's attention read needs every row, for the PLE
|
||||
embedding is computed for all of them."""
|
||||
embedding is computed for all of them. ``linear_attention`` for a GDN
|
||||
layer."""
|
||||
if residual is None:
|
||||
residual = GatedResidualState.facts(
|
||||
attn_reads_every_row=_has_ple(layer_id, config)
|
||||
)
|
||||
sparse = _qwen3_5_is_moe(config)
|
||||
return (
|
||||
declare_attn(read=residual.attn_readout, update=residual.attn_update),
|
||||
declare_attn(
|
||||
read=residual.attn_readout,
|
||||
update=residual.attn_update,
|
||||
**(_linear_attn_options() if linear_attention else {}),
|
||||
),
|
||||
declare_ffn(
|
||||
sparse=sparse,
|
||||
next_layer_sparse=sparse,
|
||||
@@ -1467,7 +1502,7 @@ def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
|
||||
)
|
||||
|
||||
|
||||
def _build_qwen4_exp_stages(config, layer_id, residual):
|
||||
def _build_qwen4_exp_stages(config, layer_id, residual, *, linear_attention=False):
|
||||
"""The attention and FFN stage boundaries of one gated hyper-connection
|
||||
layer, declared by _qwen4_exp_stage_facts with the layer's own reads and
|
||||
updates (``residual``).
|
||||
@@ -1476,7 +1511,9 @@ def _build_qwen4_exp_stages(config, layer_id, residual):
|
||||
the writes are gated injections, so neither stage offers a fused
|
||||
add-and-norm candidate.
|
||||
"""
|
||||
attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual)
|
||||
attn, ffn = _qwen4_exp_stage_facts(
|
||||
config, layer_id, residual=residual, linear_attention=linear_attention
|
||||
)
|
||||
return append_stages((attn, None), (ffn, None))
|
||||
|
||||
|
||||
@@ -1489,6 +1526,7 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
linear_attention: bool = False,
|
||||
) -> None:
|
||||
self.hc_count = config.hc_count
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -1549,6 +1587,7 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
# The PLE embedding is computed for every row.
|
||||
attn_reads_every_row=_has_ple(layer_id, config),
|
||||
).residual_ops(),
|
||||
linear_attention=linear_attention,
|
||||
)
|
||||
|
||||
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
|
||||
@@ -1639,6 +1678,12 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch")
|
||||
self.ple.forward_idle(forward_batch)
|
||||
return residual
|
||||
if _shards_rows_over_cp(forward_batch):
|
||||
# The n-gram window and the short conv read the sequence in global
|
||||
# token order: run on the gathered rows, keep this rank's rows.
|
||||
full_rows = cp_gather_after_forward(residual, forward_batch)
|
||||
embedding = self.ple(full_rows, forward_batch, ple_batch)
|
||||
return residual + cp_shard_hidden_states(embedding, forward_batch)
|
||||
return residual + self.ple(residual, forward_batch, ple_batch)
|
||||
|
||||
def _prepare_attn_stage(
|
||||
@@ -1691,6 +1736,8 @@ class Qwen4ExpLayerExtensionMixin:
|
||||
class Qwen4ExpLinearDecoderLayer(
|
||||
Qwen4ExpLayerExtensionMixin, Qwen3_5LinearDecoderLayer
|
||||
):
|
||||
stage_facts = staticmethod(partial(_qwen4_exp_stage_facts, linear_attention=True))
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
@@ -1709,7 +1756,9 @@ class Qwen4ExpLinearDecoderLayer(
|
||||
is_nextn,
|
||||
build_stages=False,
|
||||
)
|
||||
self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix)
|
||||
self._init_qwen4_exp_layer_extensions(
|
||||
config, layer_id, quant_config, prefix, linear_attention=True
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -1814,11 +1863,22 @@ class Qwen4ExpAttentionDecoderLayer(
|
||||
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
|
||||
),
|
||||
)
|
||||
indexer_kwargs = {}
|
||||
if _shards_rows_over_cp(forward_batch):
|
||||
# Prefill CP: this rank's rows select against the index keys of the
|
||||
# whole sequence, gathered in token order with the RoPE positions
|
||||
# the model split (the batch keeps them whole).
|
||||
indexer_kwargs["cp_global_rope_positions"] = (
|
||||
forward_batch.mrope_positions
|
||||
if positions.ndim == 2
|
||||
else forward_batch.positions
|
||||
)
|
||||
topk_indices = self.indexer(
|
||||
hidden_states,
|
||||
positions,
|
||||
forward_batch,
|
||||
indexer_metadata,
|
||||
**indexer_kwargs,
|
||||
)
|
||||
should_capture = getattr(
|
||||
sparse_backend, "should_capture_mtp_sparse_indices", None
|
||||
@@ -1984,6 +2044,12 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
pp_proxy_tensors, forward_batch
|
||||
)
|
||||
|
||||
shards_rows = _shards_rows_over_cp(forward_batch)
|
||||
if shards_rows:
|
||||
hidden_states, positions = cp_split_before_forward(
|
||||
hidden_states, positions, forward_batch
|
||||
)
|
||||
|
||||
breakable_ple = (
|
||||
self.has_ple
|
||||
and is_in_breakable_cuda_graph()
|
||||
@@ -2060,6 +2126,15 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
|
||||
forward_batch,
|
||||
_MixStreams(self.hyper_connection_mixer, normalized_input),
|
||||
)
|
||||
if shards_rows:
|
||||
hidden_states = cp_gather_after_forward(hidden_states, forward_batch)
|
||||
# Only hidden-state capture reads the streams; gather them for it.
|
||||
capture = forward_batch.capture_hidden_mode
|
||||
hc_hidden_states = (
|
||||
cp_gather_after_forward(hc_hidden_states, forward_batch)
|
||||
if capture is not None and capture.need_capture()
|
||||
else None
|
||||
)
|
||||
if not forward_batch.forward_mode.is_idle():
|
||||
return hidden_states, hc_hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user