[Qwen3.8 CP 4/4] Prefill context parallel E2E support of Qwen3.8 (#39724)

This commit is contained in:
Shunkangz
2026-10-13 23:19:46 -07:00
committed by GitHub
parent 7ca4cbfcef
commit 0d130f1915
16 changed files with 640 additions and 62 deletions
@@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
# Architectures whose model code implements CP-TP group sharing: the CP group
@@ -17,7 +18,10 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha
# MoE and linear attention keep their TP partition (the linear attention
# gathers the sequence where its recurrence needs it). Model ports register
# here; every other model keeps the ordinary prefill CP path.
_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"}
_SUPPORTED_MODELS: set[str] = {
"Glm5NextForConditionalGeneration",
"Qwen4ExpForConditionalGeneration",
}
def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
@@ -44,25 +48,50 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
# This hook runs before speculative defaults and NEXTN alias resolution.
# Native MTP keeps the target's CP/TP placement; other draft architectures
# have their own topology and model-boundary contracts.
if cfg.speculative_algorithm is not None and (
model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration"
or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
or cfg.speculative_eagle_topk not in (None, 1)
):
raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.")
# have their own topology and model-boundary contracts. Only GLM-5.3-Flash
# has its MTP ported.
if cfg.speculative_algorithm is not None:
if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration":
raise ValueError(
"--speculative-algorithm is not supported with CP-TP group sharing."
)
if (
cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN")
or cfg.speculative_draft_model_path not in (None, cfg.model_path)
or cfg.speculative_eagle_topk not in (None, 1)
):
raise ValueError(
"CP/TP group sharing only supports native MTP with top-k 1."
)
unsupported = {
"enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split,
"dcp-size > 1": cfg.dcp_size > 1,
"enable-mixed-chunk": cfg.enable_mixed_chunk,
"moe-dp-size > 1": cfg.moe_dp_size > 1,
"disaggregation-mode": cfg.disaggregation_mode != "null",
}
if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration":
unsupported |= {
# The MoE runs TP experts on rows gathered over the TP group.
# Waterfill turns on DeepEP and EP after this hook.
"ep-size > 1": view.ep_size > 1,
"moe-a2a-backend": view.moe_a2a_backend != "none",
"enable-waterfill": cfg.enable_waterfill,
"pp-size > 1": cfg.pp_size > 1,
# Prefill only: a decode node serves plain TP.
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
}
else:
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
for flag, enabled in unsupported.items():
if enabled:
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
# Decode sees this prefill as TP cp_size, while its sender stages by
# attention TP 1, so the two would pick different staging layouts.
if cfg.disaggregation_mode == "prefill" and envs.SGLANG_DISAGG_STAGING_BUFFER.get():
raise ValueError(
"SGLANG_DISAGG_STAGING_BUFFER is not supported with CP-TP group sharing."
)
linear_config = mambaish_config(model)
if getattr(linear_config, "linear_num_key_heads", None) is not None:
@@ -45,6 +45,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_schedule,
get_serving,
linear_attn_tp_size,
max_prefill_buffer_tokens,
)
from sglang.srt.server_args import ServerArgs
@@ -320,6 +321,11 @@ class CommonKVManager(BaseKVManager):
self.attn_tp_rank = parallel.attn_tp_rank
self.attn_cp_size = parallel.attn_cp_size
self.attn_cp_rank = parallel.attn_cp_rank
# The TP width the transferred state is laid out over, which decode
# pairs with: attention TP, or the TP group under CP-TP group sharing,
# where the GDN heads are partitioned over it and QSA/PLE state is whole
# on every rank. KV heads keep following attention TP.
self.state_tp_size = linear_attn_tp_size()
self.dcp_size = parallel.attn_dcp_size
self.dcp_rank = parallel.attn_dcp_rank
self.attn_dp_size = parallel.attn_dp_size
@@ -339,7 +345,11 @@ class CommonKVManager(BaseKVManager):
self.is_hybrid_mla_backend
and disaggregation_mode == DisaggregationMode.DECODE
)
self.enable_all_cp_ranks_for_transfer = (
# CP-TP group sharing: CP rank r holds every KV head, the CP-replicated
# state and linear-attention head shard r. It transfers as TP rank r of
# a CP-1 prefill (see register_to_bootstrap), not as a CP replica.
cp_tp_group_sharing = parallel.enable_cp_tp_group_sharing
self.enable_all_cp_ranks_for_transfer = not cp_tp_group_sharing and (
envs.SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER.get()
or cp_sharded_prefill
or hybrid_decode_pulls_all_ranks
@@ -389,6 +399,7 @@ class CommonKVManager(BaseKVManager):
# participate in KV transfer; Otherwise only CP rank 0 sends.
self.is_dummy_cp_rank = (
not self.enable_all_cp_ranks_for_transfer
and not cp_tp_group_sharing
and self.attn_cp_size > 1
and self.attn_cp_rank != 0
)
@@ -471,6 +482,7 @@ class CommonKVManager(BaseKVManager):
self.attn_cp_size > 1
and self.attn_cp_rank != 0
and not get_parallel().enable_dsa_cache_layer_split
and not get_parallel().enable_cp_tp_group_sharing
)
def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool:
@@ -1396,11 +1408,16 @@ class CommonKVManager(BaseKVManager):
bootstrap_na = NetworkAddress(host, self.bootstrap_port)
url = f"{bootstrap_na.to_url()}/route"
attn_tp = (self.attn_tp_size, self.attn_tp_rank)
attn_cp = (self.attn_cp_size, self.attn_cp_rank)
if get_parallel().enable_cp_tp_group_sharing:
# Decode pairs its TP rank r with this CP rank r (see __init__).
attn_tp, attn_cp = attn_cp, (1, 0)
payload = {
"attn_tp_size": self.attn_tp_size,
"attn_tp_rank": self.attn_tp_rank,
"attn_cp_size": self.attn_cp_size,
"attn_cp_rank": self.attn_cp_rank,
"attn_tp_size": attn_tp[0],
"attn_tp_rank": attn_tp[1],
"attn_cp_size": attn_cp[0],
"attn_cp_rank": attn_cp[1],
"attn_dp_size": self.attn_dp_size,
"attn_dp_rank": self.attn_dp_rank,
"pp_size": self.pp_size,
@@ -1959,7 +1959,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
)
if (
target_rank_registration_info is not None
and self.attn_tp_size
and self.state_tp_size
!= target_rank_registration_info.dst_attn_tp_size
):
rc = (
@@ -2291,7 +2291,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
"""
logger.warning_once(
"Using Mamba state slice transfer for different runtime attention TP "
f"sizes: prefill={self.attn_tp_size}, decode={dst_attn_tp_size}. "
f"sizes: prefill={self.state_tp_size}, decode={dst_attn_tp_size}. "
"Performance may be affected."
)
assert len(prefill_mamba_index) == 1, "Mamba should have single state index"
@@ -2310,7 +2310,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
dst_state_item_lens,
)
local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size
local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size
dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size
transfer_blocks = []
@@ -2349,7 +2349,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_dim=src_dim,
dst_dim=dst_dim,
outer_count=outer_count,
src_attn_tp_size=self.attn_tp_size,
src_attn_tp_size=self.state_tp_size,
dst_attn_tp_size=dst_attn_tp_size,
dst_tp_rank_in_group=dst_tp_rank_in_group,
local_tp_rank_in_group=local_tp_rank_in_group,
+12 -6
View File
@@ -1654,7 +1654,7 @@ class MoriKVManager(CommonKVManager):
)
if st == "mamba":
if peer_info.decode_tp_size != self.attn_tp_size and 0 in src_dims:
if peer_info.decode_tp_size != self.state_tp_size and 0 in src_dims:
raise RuntimeError(
"Replicated Mamba PD state transfer currently requires "
"matching prefill/decode attention TP sizes"
@@ -1727,7 +1727,7 @@ class MoriKVManager(CommonKVManager):
f"got src={src_state_indices.size}, dst={dst_state_indices.size}"
)
tp_mismatch = peer_info.decode_tp_size != self.attn_tp_size
tp_mismatch = peer_info.decode_tp_size != self.state_tp_size
# If dim info missing, silently degrade to whole-item copy (Mooncake compat)
if tp_mismatch and (
@@ -1738,7 +1738,7 @@ class MoriKVManager(CommonKVManager):
if tp_mismatch:
logger.warning_once(
"Using Mamba state slice transfer for different TP sizes between prefill and decode. "
f"Prefill attn_tp_size={self.attn_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. "
f"Prefill attn_tp_size={self.state_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. "
"Performance may be affected."
)
@@ -1746,7 +1746,7 @@ class MoriKVManager(CommonKVManager):
dst_idx = int(dst_state_indices[0])
statuses: List[TransferStatus] = []
local_tp_rank = self.kv_args.engine_rank % self.attn_tp_size
local_tp_rank = self.kv_args.engine_rank % self.state_tp_size
dst_tp_rank = peer_info.decode_tp_rank % peer_info.decode_tp_size
pairs = build_transfer_entry_pairs(
@@ -1792,7 +1792,7 @@ class MoriKVManager(CommonKVManager):
src_dim=src_dim,
dst_dim=dst_dim,
outer_count=outer_count,
src_attn_tp_size=self.attn_tp_size,
src_attn_tp_size=self.state_tp_size,
dst_attn_tp_size=peer_info.decode_tp_size,
dst_tp_rank_in_group=dst_tp_rank,
local_tp_rank_in_group=local_tp_rank,
@@ -1853,7 +1853,13 @@ class MoriKVManager(CommonKVManager):
"PP>1 needs peer state_layer_ids for global-layer descriptor "
"pairing."
)
if peer_info.decode_tp_size != self.attn_tp_size:
# QSA state is whole on every rank, so it pairs by state TP.
src_tp_size = (
self.state_tp_size
if state_type in ("qsa_pending", "qsa_compressed")
else self.attn_tp_size
)
if peer_info.decode_tp_size != src_tp_size:
raise RuntimeError(
f"PD disagg: heterogeneous TP not supported for {state_type} yet."
)
+11 -5
View File
@@ -2539,7 +2539,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
"""
logger.warning_once(
"Using Mamba state slice transfer for different TP sizes. "
f"Prefill attn_tp_size={self.attn_tp_size}, "
f"Prefill attn_tp_size={self.state_tp_size}, "
f"Decode attn_tp_size={decode_tp_size}."
)
assert len(prefill_state_indices) == 1, "Mamba should have single state index"
@@ -2558,7 +2558,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
dst_layer_ids=dst_layer_ids,
)
local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size
local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size
dst_tp_rank_in_group = decode_tp_rank % decode_tp_size
src_addrs = []
@@ -2601,7 +2601,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
src_dim=src_dim,
dst_dim=dst_dim,
outer_count=outer_count,
src_attn_tp_size=self.attn_tp_size,
src_attn_tp_size=self.state_tp_size,
dst_attn_tp_size=decode_tp_size,
dst_tp_rank_in_group=dst_tp_rank_in_group,
local_tp_rank_in_group=local_tp_rank_in_group,
@@ -2761,7 +2761,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
comp_notif = f"{notif}_{i}"
if st == StateType.MAMBA:
if self.attn_tp_size != decode_tp_size:
if self.state_tp_size != decode_tp_size:
if 0 in src_dims:
raise RuntimeError(
"Replicated Mamba PD state transfer currently requires "
@@ -2856,7 +2856,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
StateType.SWA_RING,
StateType.DSV4_REQUEST_STATE,
):
if not self.is_mla_backend and self.attn_tp_size != decode_tp_size:
# QSA state is whole on every rank, so it pairs by state TP.
src_tp_size = (
self.state_tp_size
if st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED)
else self.attn_tp_size
)
if not self.is_mla_backend and src_tp_size != decode_tp_size:
raise RuntimeError(
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
)
@@ -42,6 +42,8 @@ from sglang.srt.layers.layer_boundary.ops import (
attn_cp_take_back_output,
attn_tp_gather_input,
attn_tp_slice_output,
cp_gather_in_token_order,
cp_take_back_in_token_order,
dp_cp_take_back_output,
keep_output,
moe_cp_take_back_output,
@@ -444,15 +446,31 @@ def _select_entry_step(
# A head-parallel mixer consumes full context, but its residual and
# readout coefficients remain on the CP shard. Complete the producer's
# sum onto that shard before the (possibly nonlinear) update/read.
if (
residual != residual_to
or cp_moves is None
or owes not in (None, SumGroup.ATTN_CP)
or (owes is not None and cp_moves.reduce_scatter is None)
if residual != residual_to or cp_moves is None:
raise NotImplementedError(f"{produced=} {residual=} {need=}")
if produced.in_token_order:
# The whole sequence in token order (a scan's): complete the sum on
# every row, then take this rank's shard back. No model runs two
# token-order stages back to back, so that edge is not bound.
if need.in_token_order or owes not in (
None,
SumGroup.ATTN_CP,
SumGroup.TP,
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
move = partial(cp_take_back_in_token_order, group=owes)
elif owes not in (None, SumGroup.ATTN_CP) or (
owes is not None and cp_moves.reduce_scatter is None
):
raise NotImplementedError(f"{produced=} {residual=} {need=}")
else:
move = cp_moves.reduce_scatter if owes is not None else cp_moves.take_back
local = msgspec.structs.replace(
produced, layout=residual_to, group=None, always_partial=False
produced,
layout=residual_to,
group=None,
always_partial=False,
in_token_order=False,
)
inner = _select_entry_step(
local,
@@ -466,13 +484,7 @@ def _select_entry_step(
enters_stack=enters_stack,
)
return inner._replace(
step=partial(
_move_before_read,
move=cp_moves.reduce_scatter
if owes is not None
else cp_moves.take_back,
read=inner.step,
)
step=partial(_move_before_read, move=move, read=inner.step)
)
if sliced:
# Each attention-TP rank takes its own slice: the reduce-scatter
@@ -549,6 +561,9 @@ def _select_entry_step(
cp_moves=cp_moves,
enters_stack=enters_stack,
).step
if need.in_token_order:
# The stage scans the whole sequence: gather it in token order.
return EntryStep(on_chunk, input_move=cp_gather_in_token_order)
return EntryStep(partial(cp_moves.gather, gather=on_chunk))
if gathered == {TokenAxis.ATTN_TP}:
# A complete input on each rank's slice, gathered over attention TP
@@ -290,6 +290,8 @@ def _bind_stage(
and TokenAxis.ATTN_CP
in edge.produced.layout.sharded - edge.need.layout.sharded
and not _cp_gathers_over_attn_cp()
# CP-TP group sharing is the MoE-CP gather MHC has been run with.
and not get_parallel().enable_cp_tp_group_sharing
):
raise NotImplementedError(
"an update applied at the stage's exit with a gather over attention CP"
@@ -71,6 +71,9 @@ class InputContract(msgspec.Struct, frozen=True):
gathered_by_compute: Token axes compute can gather internally; the boundary
may hand it input still sharded over those axes.
read: Operation that derives input from the updated residual.
in_token_order: Compute reads the whole sequence in the batch's token
order (a scan over it), so a gather over attention CP restores
that order instead of placing the shards side by side.
"""
layout: Layout
@@ -79,6 +82,7 @@ class InputContract(msgspec.Struct, frozen=True):
gathered_by_compute: FrozenSet[TokenAxis] = frozenset()
# How the consumer reads its input from the residual.
read: ResidualReadout = NORM_READOUT
in_token_order: bool = False
class OutputContract(msgspec.Struct, frozen=True):
@@ -99,6 +103,8 @@ class OutputContract(msgspec.Struct, frozen=True):
update: Producer residual operation; None on an arrival contract that
declares capabilities and obtains the actual update from the stream.
transform: Optional operation on the contribution before residual update.
in_token_order: The output covers the whole sequence in the batch's
token order; the consumer takes this rank's CP shard of it back.
These are permissions, not a record of a particular output. The exit
decision and ResidualStream record what that output actually owes.
@@ -118,6 +124,7 @@ class OutputContract(msgspec.Struct, frozen=True):
# description has no producer object; its edge declares capabilities only.
update: Optional[ResidualUpdate] = PLAIN_ADD
transform: Optional[OutputTransform] = None
in_token_order: bool = False
class StageContract(msgspec.Struct, frozen=True):
@@ -286,6 +286,8 @@ class StageDeclaration:
gathers_attn_tp_input: Whether attention gathers TP-sharded input itself.
tp_group: Group partitioning attention heads. Full TP consumers require
full token rows, including when TP shares the prefill CP group.
in_token_order: Whether an attention reads the whole sequence in token
order (a scan over it), e.g. linear attention under prefill CP.
dense_tp_size: Dense FFN compute width: None uses the configured width,
1 means local compute, the attention TP size means compute on the
attention's rows over attention TP, and the full TP size means TP
@@ -324,6 +326,7 @@ class StageDeclaration:
reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED
gathers_attn_tp_input: bool = False
tp_group: SumGroup = SumGroup.ATTN_TP
in_token_order: bool = False
dense_tp_size: Optional[int] = None
exit_rows: Optional[ExitRows] = None
writes_at_handoff: bool = False
@@ -419,6 +422,7 @@ def declare_attn(
output_transform=None,
tp_group=SumGroup.ATTN_TP,
attn_tp_gather=None,
in_token_order=False,
):
"""Declare attention or a mixer; construct its executable boundary later.
@@ -435,15 +439,20 @@ def declare_attn(
tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers.
attn_tp_gather: Implementation of the gather over attention TP into
this stage, tried before the boundary's own (see StageDeclaration).
in_token_order: Compute scans the whole sequence in token order. On a
CP extend its input is gathered into that order, and the next
stage's input completes the sum and takes this rank's shard back.
Requires ALWAYS_PARTIAL.
Returns:
A StageDeclaration with no norm, tensors or execution plan.
"""
if (
output_transform is not None
and reduction is not ProducerReduction.ALWAYS_PARTIAL
):
raise ValueError("an attention output transform requires ALWAYS_PARTIAL")
output_transform is not None or in_token_order
) and reduction is not ProducerReduction.ALWAYS_PARTIAL:
raise ValueError(
"an attention output transform or token-order input requires ALWAYS_PARTIAL"
)
return StageDeclaration(
StageKind.ATTENTION,
read,
@@ -453,6 +462,7 @@ def declare_attn(
gathers_attn_tp_input=gathers_attn_tp_input,
tp_group=tp_group,
attn_tp_gather=attn_tp_gather,
in_token_order=in_token_order,
)
@@ -504,7 +514,12 @@ def _reject_unsupported(stage):
rows of one edge are made where that edge binds (``_bind_stage``)."""
parallel = get_parallel()
if stage.update.applied_at_exit:
if stage.sparse and moe_gathers_over_moe_cp():
# CP-TP group sharing is the MoE-CP gather MHC has been run with.
if (
stage.sparse
and moe_gathers_over_moe_cp()
and not parallel.enable_cp_tp_group_sharing
):
raise NotImplementedError(
"an update applied at the stage's exit with a MoE gathered over "
"the MoE-CP group"
@@ -614,8 +629,19 @@ def _resolve_stage(stage, variant, arrival, following=None):
)
elif stage.tp_group is not SumGroup.ATTN_TP:
raise ValueError("attention heads must be partitioned over ATTN_TP or TP")
# On a CP extend a token-order stage takes the whole sequence.
in_token_order = stage.in_token_order and variant is BatchVariant.CONTEXT_PARALLEL
if in_token_order:
if get_parallel().attn_dp_size > 1:
raise NotImplementedError("a token-order attention under attention DP")
compute_rows = full
declaration = StageContract(
InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read),
InputContract(
compute_rows,
gathered_by_compute=gathers,
read=stage.read,
in_token_order=in_token_order,
),
OutputContract(
local if sp else compute_rows,
group=group if owes else None,
@@ -630,6 +656,7 @@ def _resolve_stage(stage, variant, arrival, following=None):
and following.kind is StageKind.ATTENTION,
update=stage.update,
transform=stage.output_transform,
in_token_order=in_token_order,
),
)
# An attention moves no residual: it keeps the rows it arrives on, except
@@ -28,6 +28,10 @@ from sglang.srt.distributed import (
from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.cp.utils import (
cp_gather_full_sequence_states,
cp_shard_hidden_states,
)
from sglang.srt.layers.dp_attention import (
attn_cp_reduce_scatter_tensor,
attn_tp_all_gather_into_tensor,
@@ -52,6 +56,7 @@ from sglang.srt.layers.layer_boundary.layout import (
SumGroup,
TokenAxis,
_cp_shard_token_rows,
_sum_group,
moe_cp_gathered_rows,
)
from sglang.srt.layers.layer_boundary.residual import ResidualUpdate
@@ -179,6 +184,28 @@ def moe_cp_gather(
return output
def cp_gather_in_token_order(
hidden_states: torch.Tensor, forward_batch: ForwardBatch
) -> torch.Tensor:
"""This rank's CP shard to the whole sequence in token order, without the
shards' padding."""
return cp_gather_full_sequence_states(hidden_states, forward_batch)
def cp_take_back_in_token_order(
hidden_states: torch.Tensor,
residual: torch.Tensor,
forward_batch: ForwardBatch,
*,
group: Optional[SumGroup],
):
"""The whole sequence in token order back to this rank's CP shard,
completing the sum it owes over ``group`` first."""
if group is not None:
hidden_states = _sum_group(group).all_reduce(hidden_states)
return cp_shard_hidden_states(hidden_states, forward_batch), residual
def attn_tp_all_reduce(
hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True
) -> torch.Tensor:
+82 -7
View File
@@ -3,6 +3,7 @@
import math
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from functools import partial
from typing import Any, Iterable, Optional, Set, Tuple, Union
import msgspec
@@ -26,6 +27,12 @@ from sglang.srt.layers.attention.linear.utils import (
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.cp.utils import (
cp_gather_after_forward,
cp_shard_hidden_states,
cp_split_before_forward,
is_cp_active,
)
from sglang.srt.layers.dp_attention import (
attn_tp_all_reduce,
dp_gather_replicate,
@@ -40,6 +47,7 @@ from sglang.srt.layers.hyperconnection import (
)
from sglang.srt.layers.layer_boundary import (
GatedResidualState,
SumGroup,
append_stages,
declare_attn,
declare_ffn,
@@ -89,7 +97,7 @@ from sglang.srt.models.qwen4_exp_ple_table import (
make_ple_file_prefetcher,
make_ple_file_rss_trimmer,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_parallel, linear_attn_parallel_group
from sglang.srt.utils import get_bool_env_var, is_hip, logger
_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip()
@@ -1444,20 +1452,47 @@ def _has_ple(layer_id, config) -> bool:
return (layer_id + 1) in config.ple_layer_ids
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
def _shards_rows_over_cp(forward_batch: ForwardBatch) -> bool:
"""A CP extend under CP-TP group sharing: every layer runs on this rank's
zigzag token rows, and the GDN and the PLE gather the sequence in global
token order where they need it. The batch metadata stays global."""
return (
get_parallel().enable_cp_tp_group_sharing
and forward_batch.attn_cp_metadata is not None
and is_cp_active(forward_batch)
)
def _linear_attn_options():
"""Linear attention scans the sequence in token order, and its output owes
the sum over the group its heads are partitioned over."""
return dict(
tp_group=(
SumGroup.TP if linear_attn_parallel_group() == "tp" else SumGroup.ATTN_TP
),
in_token_order=True,
)
def _qwen4_exp_stage_facts(config, layer_id, *, residual=None, linear_attention=False):
"""The attention and FFN stages a gated hyper-connection layer of either
kind declares: the model's shared declaration function (see make_layers),
which the layer declares itself with. ``residual`` is the layer's own
reads and updates of the streams; without it, the stages declare what
those do. A PLE layer's attention read needs every row, for the PLE
embedding is computed for all of them."""
embedding is computed for all of them. ``linear_attention`` for a GDN
layer."""
if residual is None:
residual = GatedResidualState.facts(
attn_reads_every_row=_has_ple(layer_id, config)
)
sparse = _qwen3_5_is_moe(config)
return (
declare_attn(read=residual.attn_readout, update=residual.attn_update),
declare_attn(
read=residual.attn_readout,
update=residual.attn_update,
**(_linear_attn_options() if linear_attention else {}),
),
declare_ffn(
sparse=sparse,
next_layer_sparse=sparse,
@@ -1467,7 +1502,7 @@ def _qwen4_exp_stage_facts(config, layer_id, *, residual=None):
)
def _build_qwen4_exp_stages(config, layer_id, residual):
def _build_qwen4_exp_stages(config, layer_id, residual, *, linear_attention=False):
"""The attention and FFN stage boundaries of one gated hyper-connection
layer, declared by _qwen4_exp_stage_facts with the layer's own reads and
updates (``residual``).
@@ -1476,7 +1511,9 @@ def _build_qwen4_exp_stages(config, layer_id, residual):
the writes are gated injections, so neither stage offers a fused
add-and-norm candidate.
"""
attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual)
attn, ffn = _qwen4_exp_stage_facts(
config, layer_id, residual=residual, linear_attention=linear_attention
)
return append_stages((attn, None), (ffn, None))
@@ -1489,6 +1526,7 @@ class Qwen4ExpLayerExtensionMixin:
layer_id: int,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
linear_attention: bool = False,
) -> None:
self.hc_count = config.hc_count
self.hidden_size = config.hidden_size
@@ -1549,6 +1587,7 @@ class Qwen4ExpLayerExtensionMixin:
# The PLE embedding is computed for every row.
attn_reads_every_row=_has_ple(layer_id, config),
).residual_ops(),
linear_attention=linear_attention,
)
from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm
@@ -1639,6 +1678,12 @@ class Qwen4ExpLayerExtensionMixin:
raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch")
self.ple.forward_idle(forward_batch)
return residual
if _shards_rows_over_cp(forward_batch):
# The n-gram window and the short conv read the sequence in global
# token order: run on the gathered rows, keep this rank's rows.
full_rows = cp_gather_after_forward(residual, forward_batch)
embedding = self.ple(full_rows, forward_batch, ple_batch)
return residual + cp_shard_hidden_states(embedding, forward_batch)
return residual + self.ple(residual, forward_batch, ple_batch)
def _prepare_attn_stage(
@@ -1691,6 +1736,8 @@ class Qwen4ExpLayerExtensionMixin:
class Qwen4ExpLinearDecoderLayer(
Qwen4ExpLayerExtensionMixin, Qwen3_5LinearDecoderLayer
):
stage_facts = staticmethod(partial(_qwen4_exp_stage_facts, linear_attention=True))
def __init__(
self,
config: Qwen4ExpTextConfig,
@@ -1709,7 +1756,9 @@ class Qwen4ExpLinearDecoderLayer(
is_nextn,
build_stages=False,
)
self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix)
self._init_qwen4_exp_layer_extensions(
config, layer_id, quant_config, prefix, linear_attention=True
)
def forward(
self,
@@ -1814,11 +1863,22 @@ class Qwen4ExpAttentionDecoderLayer(
forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist()
),
)
indexer_kwargs = {}
if _shards_rows_over_cp(forward_batch):
# Prefill CP: this rank's rows select against the index keys of the
# whole sequence, gathered in token order with the RoPE positions
# the model split (the batch keeps them whole).
indexer_kwargs["cp_global_rope_positions"] = (
forward_batch.mrope_positions
if positions.ndim == 2
else forward_batch.positions
)
topk_indices = self.indexer(
hidden_states,
positions,
forward_batch,
indexer_metadata,
**indexer_kwargs,
)
should_capture = getattr(
sparse_backend, "should_capture_mtp_sparse_indices", None
@@ -1984,6 +2044,12 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
pp_proxy_tensors, forward_batch
)
shards_rows = _shards_rows_over_cp(forward_batch)
if shards_rows:
hidden_states, positions = cp_split_before_forward(
hidden_states, positions, forward_batch
)
breakable_ple = (
self.has_ple
and is_in_breakable_cuda_graph()
@@ -2060,6 +2126,15 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM):
forward_batch,
_MixStreams(self.hyper_connection_mixer, normalized_input),
)
if shards_rows:
hidden_states = cp_gather_after_forward(hidden_states, forward_batch)
# Only hidden-state capture reads the streams; gather them for it.
capture = forward_batch.capture_hidden_mode
hc_hidden_states = (
cp_gather_after_forward(hc_hidden_states, forward_batch)
if capture is not None and capture.need_capture()
else None
)
if not forward_batch.forward_mode.is_idle():
return hidden_states, hc_hidden_states