diff --git a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py index e76963b36e..8a1b51e0ad 100644 --- a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py +++ b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py @@ -10,6 +10,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.configs.hybrid_arch import mambaish_config +from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase # Architectures whose model code implements CP-TP group sharing: the CP group @@ -17,7 +18,10 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha # MoE and linear attention keep their TP partition (the linear attention # gathers the sequence where its recurrence needs it). Model ports register # here; every other model keeps the ordinary prefill CP path. -_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"} +_SUPPORTED_MODELS: set[str] = { + "Glm5NextForConditionalGeneration", + "Qwen4ExpForConditionalGeneration", +} def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: @@ -44,25 +48,50 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: # This hook runs before speculative defaults and NEXTN alias resolution. # Native MTP keeps the target's CP/TP placement; other draft architectures - # have their own topology and model-boundary contracts. - if cfg.speculative_algorithm is not None and ( - model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration" - or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN") - or cfg.speculative_draft_model_path not in (None, cfg.model_path) - or cfg.speculative_eagle_topk not in (None, 1) - ): - raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.") + # have their own topology and model-boundary contracts. Only GLM-5.3-Flash + # has its MTP ported. + if cfg.speculative_algorithm is not None: + if model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration": + raise ValueError( + "--speculative-algorithm is not supported with CP-TP group sharing." + ) + if ( + cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN") + or cfg.speculative_draft_model_path not in (None, cfg.model_path) + or cfg.speculative_eagle_topk not in (None, 1) + ): + raise ValueError( + "CP/TP group sharing only supports native MTP with top-k 1." + ) unsupported = { "enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split, "dcp-size > 1": cfg.dcp_size > 1, "enable-mixed-chunk": cfg.enable_mixed_chunk, "moe-dp-size > 1": cfg.moe_dp_size > 1, - "disaggregation-mode": cfg.disaggregation_mode != "null", } + if model.hf_config.architectures[0] == "Qwen4ExpForConditionalGeneration": + unsupported |= { + # The MoE runs TP experts on rows gathered over the TP group. + # Waterfill turns on DeepEP and EP after this hook. + "ep-size > 1": view.ep_size > 1, + "moe-a2a-backend": view.moe_a2a_backend != "none", + "enable-waterfill": cfg.enable_waterfill, + "pp-size > 1": cfg.pp_size > 1, + # Prefill only: a decode node serves plain TP. + "disaggregation-mode decode": cfg.disaggregation_mode == "decode", + } + else: + unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null" for flag, enabled in unsupported.items(): if enabled: raise ValueError(f"--{flag} is not supported with CP-TP group sharing.") + # Decode sees this prefill as TP cp_size, while its sender stages by + # attention TP 1, so the two would pick different staging layouts. + if cfg.disaggregation_mode == "prefill" and envs.SGLANG_DISAGG_STAGING_BUFFER.get(): + raise ValueError( + "SGLANG_DISAGG_STAGING_BUFFER is not supported with CP-TP group sharing." + ) linear_config = mambaish_config(model) if getattr(linear_config, "linear_num_key_heads", None) is not None: diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index a6834df2c5..1af1f61af1 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -45,6 +45,7 @@ from sglang.srt.runtime_context import ( get_parallel, get_schedule, get_serving, + linear_attn_tp_size, max_prefill_buffer_tokens, ) from sglang.srt.server_args import ServerArgs @@ -320,6 +321,11 @@ class CommonKVManager(BaseKVManager): self.attn_tp_rank = parallel.attn_tp_rank self.attn_cp_size = parallel.attn_cp_size self.attn_cp_rank = parallel.attn_cp_rank + # The TP width the transferred state is laid out over, which decode + # pairs with: attention TP, or the TP group under CP-TP group sharing, + # where the GDN heads are partitioned over it and QSA/PLE state is whole + # on every rank. KV heads keep following attention TP. + self.state_tp_size = linear_attn_tp_size() self.dcp_size = parallel.attn_dcp_size self.dcp_rank = parallel.attn_dcp_rank self.attn_dp_size = parallel.attn_dp_size @@ -339,7 +345,11 @@ class CommonKVManager(BaseKVManager): self.is_hybrid_mla_backend and disaggregation_mode == DisaggregationMode.DECODE ) - self.enable_all_cp_ranks_for_transfer = ( + # CP-TP group sharing: CP rank r holds every KV head, the CP-replicated + # state and linear-attention head shard r. It transfers as TP rank r of + # a CP-1 prefill (see register_to_bootstrap), not as a CP replica. + cp_tp_group_sharing = parallel.enable_cp_tp_group_sharing + self.enable_all_cp_ranks_for_transfer = not cp_tp_group_sharing and ( envs.SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER.get() or cp_sharded_prefill or hybrid_decode_pulls_all_ranks @@ -389,6 +399,7 @@ class CommonKVManager(BaseKVManager): # participate in KV transfer; Otherwise only CP rank 0 sends. self.is_dummy_cp_rank = ( not self.enable_all_cp_ranks_for_transfer + and not cp_tp_group_sharing and self.attn_cp_size > 1 and self.attn_cp_rank != 0 ) @@ -471,6 +482,7 @@ class CommonKVManager(BaseKVManager): self.attn_cp_size > 1 and self.attn_cp_rank != 0 and not get_parallel().enable_dsa_cache_layer_split + and not get_parallel().enable_cp_tp_group_sharing ) def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool: @@ -1396,11 +1408,16 @@ class CommonKVManager(BaseKVManager): bootstrap_na = NetworkAddress(host, self.bootstrap_port) url = f"{bootstrap_na.to_url()}/route" + attn_tp = (self.attn_tp_size, self.attn_tp_rank) + attn_cp = (self.attn_cp_size, self.attn_cp_rank) + if get_parallel().enable_cp_tp_group_sharing: + # Decode pairs its TP rank r with this CP rank r (see __init__). + attn_tp, attn_cp = attn_cp, (1, 0) payload = { - "attn_tp_size": self.attn_tp_size, - "attn_tp_rank": self.attn_tp_rank, - "attn_cp_size": self.attn_cp_size, - "attn_cp_rank": self.attn_cp_rank, + "attn_tp_size": attn_tp[0], + "attn_tp_rank": attn_tp[1], + "attn_cp_size": attn_cp[0], + "attn_cp_rank": attn_cp[1], "attn_dp_size": self.attn_dp_size, "attn_dp_rank": self.attn_dp_rank, "pp_size": self.pp_size, diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 04714abf24..7b30cf6c01 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1959,7 +1959,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): ) if ( target_rank_registration_info is not None - and self.attn_tp_size + and self.state_tp_size != target_rank_registration_info.dst_attn_tp_size ): rc = ( @@ -2291,7 +2291,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): """ logger.warning_once( "Using Mamba state slice transfer for different runtime attention TP " - f"sizes: prefill={self.attn_tp_size}, decode={dst_attn_tp_size}. " + f"sizes: prefill={self.state_tp_size}, decode={dst_attn_tp_size}. " "Performance may be affected." ) assert len(prefill_mamba_index) == 1, "Mamba should have single state index" @@ -2310,7 +2310,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): dst_state_item_lens, ) - local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size + local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size transfer_blocks = [] @@ -2349,7 +2349,7 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): src_dim=src_dim, dst_dim=dst_dim, outer_count=outer_count, - src_attn_tp_size=self.attn_tp_size, + src_attn_tp_size=self.state_tp_size, dst_attn_tp_size=dst_attn_tp_size, dst_tp_rank_in_group=dst_tp_rank_in_group, local_tp_rank_in_group=local_tp_rank_in_group, diff --git a/python/sglang/srt/disaggregation/mori/conn.py b/python/sglang/srt/disaggregation/mori/conn.py index 4bedf63c61..6ba0f99bf3 100644 --- a/python/sglang/srt/disaggregation/mori/conn.py +++ b/python/sglang/srt/disaggregation/mori/conn.py @@ -1654,7 +1654,7 @@ class MoriKVManager(CommonKVManager): ) if st == "mamba": - if peer_info.decode_tp_size != self.attn_tp_size and 0 in src_dims: + if peer_info.decode_tp_size != self.state_tp_size and 0 in src_dims: raise RuntimeError( "Replicated Mamba PD state transfer currently requires " "matching prefill/decode attention TP sizes" @@ -1727,7 +1727,7 @@ class MoriKVManager(CommonKVManager): f"got src={src_state_indices.size}, dst={dst_state_indices.size}" ) - tp_mismatch = peer_info.decode_tp_size != self.attn_tp_size + tp_mismatch = peer_info.decode_tp_size != self.state_tp_size # If dim info missing, silently degrade to whole-item copy (Mooncake compat) if tp_mismatch and ( @@ -1738,7 +1738,7 @@ class MoriKVManager(CommonKVManager): if tp_mismatch: logger.warning_once( "Using Mamba state slice transfer for different TP sizes between prefill and decode. " - f"Prefill attn_tp_size={self.attn_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. " + f"Prefill attn_tp_size={self.state_tp_size}, Decode attn_tp_size={peer_info.decode_tp_size}. " "Performance may be affected." ) @@ -1746,7 +1746,7 @@ class MoriKVManager(CommonKVManager): dst_idx = int(dst_state_indices[0]) statuses: List[TransferStatus] = [] - local_tp_rank = self.kv_args.engine_rank % self.attn_tp_size + local_tp_rank = self.kv_args.engine_rank % self.state_tp_size dst_tp_rank = peer_info.decode_tp_rank % peer_info.decode_tp_size pairs = build_transfer_entry_pairs( @@ -1792,7 +1792,7 @@ class MoriKVManager(CommonKVManager): src_dim=src_dim, dst_dim=dst_dim, outer_count=outer_count, - src_attn_tp_size=self.attn_tp_size, + src_attn_tp_size=self.state_tp_size, dst_attn_tp_size=peer_info.decode_tp_size, dst_tp_rank_in_group=dst_tp_rank, local_tp_rank_in_group=local_tp_rank, @@ -1853,7 +1853,13 @@ class MoriKVManager(CommonKVManager): "PP>1 needs peer state_layer_ids for global-layer descriptor " "pairing." ) - if peer_info.decode_tp_size != self.attn_tp_size: + # QSA state is whole on every rank, so it pairs by state TP. + src_tp_size = ( + self.state_tp_size + if state_type in ("qsa_pending", "qsa_compressed") + else self.attn_tp_size + ) + if peer_info.decode_tp_size != src_tp_size: raise RuntimeError( f"PD disagg: heterogeneous TP not supported for {state_type} yet." ) diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index fa8bc59a34..ac34e9da54 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -2539,7 +2539,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): """ logger.warning_once( "Using Mamba state slice transfer for different TP sizes. " - f"Prefill attn_tp_size={self.attn_tp_size}, " + f"Prefill attn_tp_size={self.state_tp_size}, " f"Decode attn_tp_size={decode_tp_size}." ) assert len(prefill_state_indices) == 1, "Mamba should have single state index" @@ -2558,7 +2558,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): dst_layer_ids=dst_layer_ids, ) - local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size + local_tp_rank_in_group = self.kv_args.engine_rank % self.state_tp_size dst_tp_rank_in_group = decode_tp_rank % decode_tp_size src_addrs = [] @@ -2601,7 +2601,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_dim=src_dim, dst_dim=dst_dim, outer_count=outer_count, - src_attn_tp_size=self.attn_tp_size, + src_attn_tp_size=self.state_tp_size, dst_attn_tp_size=decode_tp_size, dst_tp_rank_in_group=dst_tp_rank_in_group, local_tp_rank_in_group=local_tp_rank_in_group, @@ -2761,7 +2761,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): comp_notif = f"{notif}_{i}" if st == StateType.MAMBA: - if self.attn_tp_size != decode_tp_size: + if self.state_tp_size != decode_tp_size: if 0 in src_dims: raise RuntimeError( "Replicated Mamba PD state transfer currently requires " @@ -2856,7 +2856,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): StateType.SWA_RING, StateType.DSV4_REQUEST_STATE, ): - if not self.is_mla_backend and self.attn_tp_size != decode_tp_size: + # QSA state is whole on every rank, so it pairs by state TP. + src_tp_size = ( + self.state_tp_size + if st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED) + else self.attn_tp_size + ) + if not self.is_mla_backend and src_tp_size != decode_tp_size: raise RuntimeError( f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet." ) diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index e12aa83af7..62d66f5061 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -42,6 +42,8 @@ from sglang.srt.layers.layer_boundary.ops import ( attn_cp_take_back_output, attn_tp_gather_input, attn_tp_slice_output, + cp_gather_in_token_order, + cp_take_back_in_token_order, dp_cp_take_back_output, keep_output, moe_cp_take_back_output, @@ -444,15 +446,31 @@ def _select_entry_step( # A head-parallel mixer consumes full context, but its residual and # readout coefficients remain on the CP shard. Complete the producer's # sum onto that shard before the (possibly nonlinear) update/read. - if ( - residual != residual_to - or cp_moves is None - or owes not in (None, SumGroup.ATTN_CP) - or (owes is not None and cp_moves.reduce_scatter is None) + if residual != residual_to or cp_moves is None: + raise NotImplementedError(f"{produced=} {residual=} {need=}") + if produced.in_token_order: + # The whole sequence in token order (a scan's): complete the sum on + # every row, then take this rank's shard back. No model runs two + # token-order stages back to back, so that edge is not bound. + if need.in_token_order or owes not in ( + None, + SumGroup.ATTN_CP, + SumGroup.TP, + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + move = partial(cp_take_back_in_token_order, group=owes) + elif owes not in (None, SumGroup.ATTN_CP) or ( + owes is not None and cp_moves.reduce_scatter is None ): raise NotImplementedError(f"{produced=} {residual=} {need=}") + else: + move = cp_moves.reduce_scatter if owes is not None else cp_moves.take_back local = msgspec.structs.replace( - produced, layout=residual_to, group=None, always_partial=False + produced, + layout=residual_to, + group=None, + always_partial=False, + in_token_order=False, ) inner = _select_entry_step( local, @@ -466,13 +484,7 @@ def _select_entry_step( enters_stack=enters_stack, ) return inner._replace( - step=partial( - _move_before_read, - move=cp_moves.reduce_scatter - if owes is not None - else cp_moves.take_back, - read=inner.step, - ) + step=partial(_move_before_read, move=move, read=inner.step) ) if sliced: # Each attention-TP rank takes its own slice: the reduce-scatter @@ -549,6 +561,9 @@ def _select_entry_step( cp_moves=cp_moves, enters_stack=enters_stack, ).step + if need.in_token_order: + # The stage scans the whole sequence: gather it in token order. + return EntryStep(on_chunk, input_move=cp_gather_in_token_order) return EntryStep(partial(cp_moves.gather, gather=on_chunk)) if gathered == {TokenAxis.ATTN_TP}: # A complete input on each rank's slice, gathered over attention TP diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index eb5aab31ab..0e5b210616 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -290,6 +290,8 @@ def _bind_stage( and TokenAxis.ATTN_CP in edge.produced.layout.sharded - edge.need.layout.sharded and not _cp_gathers_over_attn_cp() + # CP-TP group sharing is the MoE-CP gather MHC has been run with. + and not get_parallel().enable_cp_tp_group_sharing ): raise NotImplementedError( "an update applied at the stage's exit with a gather over attention CP" diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 85740356b7..580c07178a 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -71,6 +71,9 @@ class InputContract(msgspec.Struct, frozen=True): gathered_by_compute: Token axes compute can gather internally; the boundary may hand it input still sharded over those axes. read: Operation that derives input from the updated residual. + in_token_order: Compute reads the whole sequence in the batch's token + order (a scan over it), so a gather over attention CP restores + that order instead of placing the shards side by side. """ layout: Layout @@ -79,6 +82,7 @@ class InputContract(msgspec.Struct, frozen=True): gathered_by_compute: FrozenSet[TokenAxis] = frozenset() # How the consumer reads its input from the residual. read: ResidualReadout = NORM_READOUT + in_token_order: bool = False class OutputContract(msgspec.Struct, frozen=True): @@ -99,6 +103,8 @@ class OutputContract(msgspec.Struct, frozen=True): update: Producer residual operation; None on an arrival contract that declares capabilities and obtains the actual update from the stream. transform: Optional operation on the contribution before residual update. + in_token_order: The output covers the whole sequence in the batch's + token order; the consumer takes this rank's CP shard of it back. These are permissions, not a record of a particular output. The exit decision and ResidualStream record what that output actually owes. @@ -118,6 +124,7 @@ class OutputContract(msgspec.Struct, frozen=True): # description has no producer object; its edge declares capabilities only. update: Optional[ResidualUpdate] = PLAIN_ADD transform: Optional[OutputTransform] = None + in_token_order: bool = False class StageContract(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index caa6c01f48..1d3326b6a4 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -286,6 +286,8 @@ class StageDeclaration: gathers_attn_tp_input: Whether attention gathers TP-sharded input itself. tp_group: Group partitioning attention heads. Full TP consumers require full token rows, including when TP shares the prefill CP group. + in_token_order: Whether an attention reads the whole sequence in token + order (a scan over it), e.g. linear attention under prefill CP. dense_tp_size: Dense FFN compute width: None uses the configured width, 1 means local compute, the attention TP size means compute on the attention's rows over attention TP, and the full TP size means TP @@ -324,6 +326,7 @@ class StageDeclaration: reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED gathers_attn_tp_input: bool = False tp_group: SumGroup = SumGroup.ATTN_TP + in_token_order: bool = False dense_tp_size: Optional[int] = None exit_rows: Optional[ExitRows] = None writes_at_handoff: bool = False @@ -419,6 +422,7 @@ def declare_attn( output_transform=None, tp_group=SumGroup.ATTN_TP, attn_tp_gather=None, + in_token_order=False, ): """Declare attention or a mixer; construct its executable boundary later. @@ -435,15 +439,20 @@ def declare_attn( tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers. attn_tp_gather: Implementation of the gather over attention TP into this stage, tried before the boundary's own (see StageDeclaration). + in_token_order: Compute scans the whole sequence in token order. On a + CP extend its input is gathered into that order, and the next + stage's input completes the sum and takes this rank's shard back. + Requires ALWAYS_PARTIAL. Returns: A StageDeclaration with no norm, tensors or execution plan. """ if ( - output_transform is not None - and reduction is not ProducerReduction.ALWAYS_PARTIAL - ): - raise ValueError("an attention output transform requires ALWAYS_PARTIAL") + output_transform is not None or in_token_order + ) and reduction is not ProducerReduction.ALWAYS_PARTIAL: + raise ValueError( + "an attention output transform or token-order input requires ALWAYS_PARTIAL" + ) return StageDeclaration( StageKind.ATTENTION, read, @@ -453,6 +462,7 @@ def declare_attn( gathers_attn_tp_input=gathers_attn_tp_input, tp_group=tp_group, attn_tp_gather=attn_tp_gather, + in_token_order=in_token_order, ) @@ -504,7 +514,12 @@ def _reject_unsupported(stage): rows of one edge are made where that edge binds (``_bind_stage``).""" parallel = get_parallel() if stage.update.applied_at_exit: - if stage.sparse and moe_gathers_over_moe_cp(): + # CP-TP group sharing is the MoE-CP gather MHC has been run with. + if ( + stage.sparse + and moe_gathers_over_moe_cp() + and not parallel.enable_cp_tp_group_sharing + ): raise NotImplementedError( "an update applied at the stage's exit with a MoE gathered over " "the MoE-CP group" @@ -614,8 +629,19 @@ def _resolve_stage(stage, variant, arrival, following=None): ) elif stage.tp_group is not SumGroup.ATTN_TP: raise ValueError("attention heads must be partitioned over ATTN_TP or TP") + # On a CP extend a token-order stage takes the whole sequence. + in_token_order = stage.in_token_order and variant is BatchVariant.CONTEXT_PARALLEL + if in_token_order: + if get_parallel().attn_dp_size > 1: + raise NotImplementedError("a token-order attention under attention DP") + compute_rows = full declaration = StageContract( - InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read), + InputContract( + compute_rows, + gathered_by_compute=gathers, + read=stage.read, + in_token_order=in_token_order, + ), OutputContract( local if sp else compute_rows, group=group if owes else None, @@ -630,6 +656,7 @@ def _resolve_stage(stage, variant, arrival, following=None): and following.kind is StageKind.ATTENTION, update=stage.update, transform=stage.output_transform, + in_token_order=in_token_order, ), ) # An attention moves no residual: it keeps the rows it arrives on, except diff --git a/python/sglang/srt/layers/layer_boundary/ops.py b/python/sglang/srt/layers/layer_boundary/ops.py index e0d7a8a208..d5ee3acb08 100644 --- a/python/sglang/srt/layers/layer_boundary/ops.py +++ b/python/sglang/srt/layers/layer_boundary/ops.py @@ -28,6 +28,10 @@ from sglang.srt.distributed import ( from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) +from sglang.srt.layers.cp.utils import ( + cp_gather_full_sequence_states, + cp_shard_hidden_states, +) from sglang.srt.layers.dp_attention import ( attn_cp_reduce_scatter_tensor, attn_tp_all_gather_into_tensor, @@ -52,6 +56,7 @@ from sglang.srt.layers.layer_boundary.layout import ( SumGroup, TokenAxis, _cp_shard_token_rows, + _sum_group, moe_cp_gathered_rows, ) from sglang.srt.layers.layer_boundary.residual import ResidualUpdate @@ -179,6 +184,28 @@ def moe_cp_gather( return output +def cp_gather_in_token_order( + hidden_states: torch.Tensor, forward_batch: ForwardBatch +) -> torch.Tensor: + """This rank's CP shard to the whole sequence in token order, without the + shards' padding.""" + return cp_gather_full_sequence_states(hidden_states, forward_batch) + + +def cp_take_back_in_token_order( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + *, + group: Optional[SumGroup], +): + """The whole sequence in token order back to this rank's CP shard, + completing the sum it owes over ``group`` first.""" + if group is not None: + hidden_states = _sum_group(group).all_reduce(hidden_states) + return cp_shard_hidden_states(hidden_states, forward_batch), residual + + def attn_tp_all_reduce( hidden_states: torch.Tensor, forward_batch: ForwardBatch, may_quantize: bool = True ) -> torch.Tensor: diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 7dbf76c514..ab9fc38d6f 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -3,6 +3,7 @@ import math from contextlib import contextmanager, nullcontext from dataclasses import dataclass +from functools import partial from typing import Any, Iterable, Optional, Set, Tuple, Union import msgspec @@ -26,6 +27,12 @@ from sglang.srt.layers.attention.linear.utils import ( select_verify_intermediate_state_indices, ) from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList +from sglang.srt.layers.cp.utils import ( + cp_gather_after_forward, + cp_shard_hidden_states, + cp_split_before_forward, + is_cp_active, +) from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, dp_gather_replicate, @@ -40,6 +47,7 @@ from sglang.srt.layers.hyperconnection import ( ) from sglang.srt.layers.layer_boundary import ( GatedResidualState, + SumGroup, append_stages, declare_attn, declare_ffn, @@ -89,7 +97,7 @@ from sglang.srt.models.qwen4_exp_ple_table import ( make_ple_file_prefetcher, make_ple_file_rss_trimmer, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, linear_attn_parallel_group from sglang.srt.utils import get_bool_env_var, is_hip, logger _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() @@ -1444,20 +1452,47 @@ def _has_ple(layer_id, config) -> bool: return (layer_id + 1) in config.ple_layer_ids -def _qwen4_exp_stage_facts(config, layer_id, *, residual=None): +def _shards_rows_over_cp(forward_batch: ForwardBatch) -> bool: + """A CP extend under CP-TP group sharing: every layer runs on this rank's + zigzag token rows, and the GDN and the PLE gather the sequence in global + token order where they need it. The batch metadata stays global.""" + return ( + get_parallel().enable_cp_tp_group_sharing + and forward_batch.attn_cp_metadata is not None + and is_cp_active(forward_batch) + ) + + +def _linear_attn_options(): + """Linear attention scans the sequence in token order, and its output owes + the sum over the group its heads are partitioned over.""" + return dict( + tp_group=( + SumGroup.TP if linear_attn_parallel_group() == "tp" else SumGroup.ATTN_TP + ), + in_token_order=True, + ) + + +def _qwen4_exp_stage_facts(config, layer_id, *, residual=None, linear_attention=False): """The attention and FFN stages a gated hyper-connection layer of either kind declares: the model's shared declaration function (see make_layers), which the layer declares itself with. ``residual`` is the layer's own reads and updates of the streams; without it, the stages declare what those do. A PLE layer's attention read needs every row, for the PLE - embedding is computed for all of them.""" + embedding is computed for all of them. ``linear_attention`` for a GDN + layer.""" if residual is None: residual = GatedResidualState.facts( attn_reads_every_row=_has_ple(layer_id, config) ) sparse = _qwen3_5_is_moe(config) return ( - declare_attn(read=residual.attn_readout, update=residual.attn_update), + declare_attn( + read=residual.attn_readout, + update=residual.attn_update, + **(_linear_attn_options() if linear_attention else {}), + ), declare_ffn( sparse=sparse, next_layer_sparse=sparse, @@ -1467,7 +1502,7 @@ def _qwen4_exp_stage_facts(config, layer_id, *, residual=None): ) -def _build_qwen4_exp_stages(config, layer_id, residual): +def _build_qwen4_exp_stages(config, layer_id, residual, *, linear_attention=False): """The attention and FFN stage boundaries of one gated hyper-connection layer, declared by _qwen4_exp_stage_facts with the layer's own reads and updates (``residual``). @@ -1476,7 +1511,9 @@ def _build_qwen4_exp_stages(config, layer_id, residual): the writes are gated injections, so neither stage offers a fused add-and-norm candidate. """ - attn, ffn = _qwen4_exp_stage_facts(config, layer_id, residual=residual) + attn, ffn = _qwen4_exp_stage_facts( + config, layer_id, residual=residual, linear_attention=linear_attention + ) return append_stages((attn, None), (ffn, None)) @@ -1489,6 +1526,7 @@ class Qwen4ExpLayerExtensionMixin: layer_id: int, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + linear_attention: bool = False, ) -> None: self.hc_count = config.hc_count self.hidden_size = config.hidden_size @@ -1549,6 +1587,7 @@ class Qwen4ExpLayerExtensionMixin: # The PLE embedding is computed for every row. attn_reads_every_row=_has_ple(layer_id, config), ).residual_ops(), + linear_attention=linear_attention, ) from sglang.srt.layers.moe.qwen4_decode import prepare_qwen4_decode_comm @@ -1639,6 +1678,12 @@ class Qwen4ExpLayerExtensionMixin: raise RuntimeError("non-idle Qwen4 PLE forward is missing its batch") self.ple.forward_idle(forward_batch) return residual + if _shards_rows_over_cp(forward_batch): + # The n-gram window and the short conv read the sequence in global + # token order: run on the gathered rows, keep this rank's rows. + full_rows = cp_gather_after_forward(residual, forward_batch) + embedding = self.ple(full_rows, forward_batch, ple_batch) + return residual + cp_shard_hidden_states(embedding, forward_batch) return residual + self.ple(residual, forward_batch, ple_batch) def _prepare_attn_stage( @@ -1691,6 +1736,8 @@ class Qwen4ExpLayerExtensionMixin: class Qwen4ExpLinearDecoderLayer( Qwen4ExpLayerExtensionMixin, Qwen3_5LinearDecoderLayer ): + stage_facts = staticmethod(partial(_qwen4_exp_stage_facts, linear_attention=True)) + def __init__( self, config: Qwen4ExpTextConfig, @@ -1709,7 +1756,9 @@ class Qwen4ExpLinearDecoderLayer( is_nextn, build_stages=False, ) - self._init_qwen4_exp_layer_extensions(config, layer_id, quant_config, prefix) + self._init_qwen4_exp_layer_extensions( + config, layer_id, quant_config, prefix, linear_attention=True + ) def forward( self, @@ -1814,11 +1863,22 @@ class Qwen4ExpAttentionDecoderLayer( forward_batch.seq_lens_cpu[: forward_batch.batch_size].tolist() ), ) + indexer_kwargs = {} + if _shards_rows_over_cp(forward_batch): + # Prefill CP: this rank's rows select against the index keys of the + # whole sequence, gathered in token order with the RoPE positions + # the model split (the batch keeps them whole). + indexer_kwargs["cp_global_rope_positions"] = ( + forward_batch.mrope_positions + if positions.ndim == 2 + else forward_batch.positions + ) topk_indices = self.indexer( hidden_states, positions, forward_batch, indexer_metadata, + **indexer_kwargs, ) should_capture = getattr( sparse_backend, "should_capture_mtp_sparse_indices", None @@ -1984,6 +2044,12 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): pp_proxy_tensors, forward_batch ) + shards_rows = _shards_rows_over_cp(forward_batch) + if shards_rows: + hidden_states, positions = cp_split_before_forward( + hidden_states, positions, forward_batch + ) + breakable_ple = ( self.has_ple and is_in_breakable_cuda_graph() @@ -2060,6 +2126,15 @@ class Qwen4ExpModel(Qwen3_5ForCausalLM): forward_batch, _MixStreams(self.hyper_connection_mixer, normalized_input), ) + if shards_rows: + hidden_states = cp_gather_after_forward(hidden_states, forward_batch) + # Only hidden-state capture reads the streams; gather them for it. + capture = forward_batch.capture_hidden_mode + hc_hidden_states = ( + cp_gather_after_forward(hc_hidden_states, forward_batch) + if capture is not None and capture.need_capture() + else None + ) if not forward_batch.forward_mode.is_idle(): return hidden_states, hc_hidden_states diff --git a/test/registered/e2e/disaggregation/test_qwen4_exp_pd_prefill_cp.py b/test/registered/e2e/disaggregation/test_qwen4_exp_pd_prefill_cp.py new file mode 100644 index 0000000000..e098973792 --- /dev/null +++ b/test/registered/e2e/disaggregation/test_qwen4_exp_pd_prefill_cp.py @@ -0,0 +1,141 @@ +"""Qwen3.8-Flash-Next (Qwen4-Exp) PD with a CP-TP group sharing prefill node. + +Prefill runs TP4 with prefill CP4 on GPUs 0-3, and the CP group is the TP +group. The residual stream, QSA attention and the QSA indexer are CP-sharded; +the MoE and the GDN keep their TP partition. Decode runs plain TP4 on GPUs +4-7 and sees only what the prefill node transferred, so an accuracy gap is the +fault of the prefill CP path or of the transfer. + +CP-TP group sharing is derived, not requested: with a wrong recipe the prefill +node boots without it and still serves. ``test_a_cp_tp_group_sharing_is_derived`` +pins the resolved flag. +""" + +import unittest + +import requests +from transformers import AutoTokenizer + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import try_cached_model + +register_cuda_ci(est_time=600, stage="nightly", runner_config="8-gpu-b200") + +MODEL = "nvidia/Qwen3.8-Flash-Next-NVFP4" +CP_SIZE = 4 + +# The recipe of test/registered/e2e/models/test_qwen4_exp_models.py, on both +# sides so the transferred GDN state has the same dtype and layout. +COMMON_ARGS = [ + "--chunked-prefill-size", + "8192", + "--linear-attn-prefill-backend", + "flashinfer", + "--linear-attn-decode-backend", + "flashinfer", + "--mamba-ssm-dtype", + "bfloat16", + "--reasoning-parser", + "qwen3-thinking", +] +PREFILL_ARGS = COMMON_ARGS + [ + "--attn-cp-size", + str(CP_SIZE), + "--enable-prefill-cp", + "--cp-strategy", + "zigzag", + # Attention TP is 1: every rank holds all attention heads and the whole + # sequence's KV. + "--mem-fraction-static", + "0.8", +] +DECODE_ARGS = COMMON_ARGS + ["--mem-fraction-static", "0.85"] + +# Past the indexer budget (2048 tokens), so QSA selects a sparse subset, and +# past the chunk size, so later chunks run CP over a cached prefix. +NEEDLE_PROMPT_TOKENS = 20000 +NEEDLE_DEPTHS = (0.1, 0.5, 0.9) +NEEDLE_KEY = "739391" + + +class TestQwen4ExpPDPrefillCP(GSM8KMixin, PDDisaggregationServerBase): + model = try_cached_model(MODEL) + prefill_tp_size = CP_SIZE + decode_tp_size = CP_SIZE + decode_base_gpu_id = CP_SIZE + extra_prefill_args = PREFILL_ARGS + extra_decode_args = DECODE_ARGS + + # The gate of the plain-TP4 sibling, test_qwen4_exp_models.py. + gsm8k_backend = "sgl_eval" + gsm8k_thinking = True + gsm8k_num_examples = 200 + gsm8k_num_threads = 32 + gsm8k_max_tokens = 16384 + gsm8k_score_threshold = 0.94 + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model) + cls.launch_all() + + def test_a_cp_tp_group_sharing_is_derived(self): + info = requests.get(self.prefill_url + "/server_info", timeout=60).json() + self.assertTrue( + info["enable_cp_tp_group_sharing"], + "CP-TP group sharing was not derived on the prefill node " + f"(attn_cp_size={info.get('attn_cp_size')}, " + f"enable_prefill_cp={info.get('enable_prefill_cp')})", + ) + self.assertEqual(info["attn_cp_size"], CP_SIZE) + self.assertEqual(info["tp_size"], CP_SIZE) + + def _needle_prompt(self, depth: float) -> str: + filler = ( + "Archive note: the weather was mild, the office lights were on, " + "and no unusual event was reported.\n" + ) + repeats = NEEDLE_PROMPT_TOKENS // len(self.tokenizer.encode(filler)) + before = int(repeats * depth) + document = ( + filler * before + + f"IMPORTANT RECORD: The access code is {NEEDLE_KEY}.\n" + + filler * (repeats - before) + ) + return self.tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "One of these records contains an access code.\n" + + document + + "\nWhat is the access code? Reply with the digits only.", + } + ], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + + def test_short_prompt(self): + # Under 2 * CP_SIZE tokens the zigzag split does not apply, so the + # prefill node runs the non-CP path, where the GDN still sums its + # TP-partitioned heads. + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + }, + timeout=600, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertIn("Paris", response.json()["text"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index 5443bea7ac..33702a5cd6 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -819,25 +819,30 @@ class TestPollCollectives(CustomTestCase): class TestCPReplicatedStateTransfer(unittest.TestCase): def test_only_nonzero_cp_ranks_without_layer_split_skip_state(self): + # Under CP-TP group sharing each CP rank owns its own linear-attention + # head shard, so no rank skips its state. cases = [ - (1, 0, False, False), - (8, 0, False, False), - (8, 1, False, True), - (8, 7, False, True), - (8, 1, True, False), + (1, 0, False, False, False), + (8, 0, False, False, False), + (8, 1, False, False, True), + (8, 7, False, False, True), + (8, 1, True, False, False), + (8, 1, False, True, False), ] - for cp_size, cp_rank, layer_split, expected in cases: + for cp_size, cp_rank, layer_split, sharing, expected in cases: with self.subTest( cp_size=cp_size, cp_rank=cp_rank, layer_split=layer_split, + sharing=sharing, ): manager = object.__new__(CommonKVManager) manager.attn_cp_size = cp_size manager.attn_cp_rank = cp_rank with get_context().override_server_args( enable_dsa_cache_layer_split=layer_split, + enable_cp_tp_group_sharing=sharing, ): self.assertEqual( manager._should_skip_cp_replicated_state_transfer(), diff --git a/test/registered/unit/disaggregation/test_register_to_bootstrap.py b/test/registered/unit/disaggregation/test_register_to_bootstrap.py index 322bca9e66..4e01acfbbf 100644 --- a/test/registered/unit/disaggregation/test_register_to_bootstrap.py +++ b/test/registered/unit/disaggregation/test_register_to_bootstrap.py @@ -239,6 +239,24 @@ class TestRegisterToBootstrap(CustomTestCase): self.assertIn(field, payload) self.assertEqual(payload["prefill_http_port"], 30000) + @patch("sglang.srt.disaggregation.common.conn.time") + @patch("sglang.srt.disaggregation.common.conn.requests.put") + def test_cp_tp_group_sharing_registers_cp_rank_as_tp_rank( + self, mock_put, mock_time + ): + # A sharing prefill CP rank holds every KV head and its own + # linear-attention shard: decode pairs with it as a CP-1 TP rank. + mock_time.monotonic.return_value = 0.0 + mock_put.return_value = MagicMock(status_code=200) + mgr = self._make_manager() + mgr.attn_cp_size, mgr.attn_cp_rank = 4, 2 + with get_context().override_server_args(enable_cp_tp_group_sharing=True): + mgr.register_to_bootstrap() + + payload = mock_put.call_args[1]["json"] + self.assertEqual([payload[k] for k in ("attn_tp_size", "attn_tp_rank")], [4, 2]) + self.assertEqual([payload[k] for k in ("attn_cp_size", "attn_cp_rank")], [1, 0]) + @patch("sglang.srt.disaggregation.common.conn.time") @patch("sglang.srt.disaggregation.common.conn.requests.put") def test_url_with_dist_init_addr(self, mock_put, mock_time): diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 376c8bcfa3..371f3e99ee 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -87,6 +87,7 @@ def parallel_of(*, attn_dp, attn_tp, attn_cp=1, **overrides): attn_cp_rank=0, moe_dense_tp_size=None, enable_prefill_cp=False, + enable_cp_tp_group_sharing=False, moe_ep_size=1, moe_tp_size=attn_dp * attn_cp * attn_tp, moe_dp_size=1, @@ -667,6 +668,18 @@ class TestMhcOnTheDeclarations(CustomTestCase): layer_case(1, 3, sparse=True, previous_sparse=True), parallel, ) + # CP-TP group sharing (Qwen4-Exp) runs MHC on the MoE-CP gather. + parallel = parallel_of( + attn_dp=1, + attn_tp=1, + attn_cp=2, + enable_prefill_cp=True, + enable_cp_tp_group_sharing=True, + ) + build_mhc( + layer_case(1, 3, sparse=True, previous_sparse=True), + parallel, + ) class TestTwoBatchOverlap(CustomTestCase): diff --git a/test/registered/unit/layer_boundary/test_token_order_attention.py b/test/registered/unit/layer_boundary/test_token_order_attention.py new file mode 100644 index 0000000000..6fda661cc7 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_token_order_attention.py @@ -0,0 +1,190 @@ +"""Linear attention under CP-TP group sharing (Qwen4-Exp): on a CP extend the +boundary gathers its input in token order, and the FFN's input completes its sum +over the TP group and takes this rank's shard back.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import test_declared_decoder_boundary as fixture +import torch + +from sglang.srt.layers.layer_boundary import ( + EdgeContract, + InputContract, + Layout, + OutputContract, + ProducerReduction, + TokenAxis, + bind_entry, + declare_attn, + layer_stack, + ops, + prepare, +) +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, CpMoves +from sglang.srt.layers.layer_boundary.layout import SumGroup +from sglang.srt.layers.layer_boundary.ops import ( + cp_gather_in_token_order, + cp_take_back_in_token_order, +) +from sglang.srt.layers.layer_boundary.residual.gated import GatedResidualState +from sglang.srt.models import qwen4_exp +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +CP = BatchVariant.CONTEXT_PARALLEL +ORDINARY = BatchVariant.ORDINARY + + +def _residual_ops(): + def unused(*args, **kwargs): + raise AssertionError("construction only") + + return GatedResidualState( + expand=unused, + attn_mix=unused, + ffn_mix=unused, + attn_combine=unused, + ffn_combine=unused, + ).residual_ops() + + +def _step(stage, variant): + return stage.plan.paths[variant].entry.prepare.keywords["step"] + + +def _build_layers(parallel, linear_group): + """A linear-attention layer, then a sparse-attention layer.""" + config = SimpleNamespace( + model_type="qwen4_exp_text", num_hidden_layers=2, ple_layer_ids=[] + ) + with ( + fixture.planning(parallel), + patch.object(qwen4_exp, "linear_attn_parallel_group", lambda: linear_group), + layer_stack(), + ): + linear = qwen4_exp._build_qwen4_exp_stages( + config, 0, _residual_ops(), linear_attention=True + ) + sparse = qwen4_exp._build_qwen4_exp_stages(config, 1, _residual_ops()) + return linear, sparse + + +class TestTokenOrderAttention(CustomTestCase): + def setUp(self): + parallel = fixture.parallel_of( + attn_dp=1, + attn_tp=1, + attn_cp=4, + enable_prefill_cp=True, + enable_cp_tp_group_sharing=True, + ) + self.linear, self.sparse = _build_layers(parallel, "tp") + + def test_linear_attention_reads_the_whole_sequence_in_token_order(self): + attn, ffn = self.linear + entry = attn.plan.paths[CP].entry + self.assertIs(_step(attn, CP).func, prepare._update_read) + self.assertIs(entry.input_move, cp_gather_in_token_order) + # The FFN's input sums over the TP group (the attention-CP group under + # sharing), takes this rank's shard back, writes it into the residual + # and reads there, then gathers for the MoE. + step = _step(ffn, CP) + self.assertIs(step.func, prepare._move_before_read) + take_back = step.keywords["move"] + self.assertIs(take_back.func, cp_take_back_in_token_order) + self.assertIs(take_back.keywords["group"], SumGroup.ATTN_CP) + read = step.keywords["read"] + self.assertIs(read.func, prepare._then_moe_cp_gather) + self.assertIs(read.keywords["gather"].func, prepare._update_read) + entry = ffn.plan.paths[CP].entry + self.assertIs(entry.declared_sum, SumGroup.ATTN_CP) + # A sum already completed for this batch only takes the shard back. + completed = entry.prepare.keywords["completed_step"].keywords["move"] + self.assertIsNone(completed.keywords["group"]) + + def test_an_ordinary_batch_completes_the_tp_sum(self): + _, ffn = self.linear + step = _step(ffn, ORDINARY) + self.assertIs(step.func, prepare._reduce_update_read) + # The attention-CP group is the TP group under sharing. + self.assertIs(step.keywords["group"], SumGroup.ATTN_CP) + + def test_sparse_attention_keeps_its_cp_shard(self): + attn, ffn = self.sparse + self.assertIsNone(attn.plan.paths[CP].entry.input_move) + self.assertIsNone(ffn.plan.paths[CP].entry.declared_sum) + self.assertIs(_step(ffn, CP).keywords["gather"].func, prepare._update_read) + + def test_without_sharing_linear_attention_binds_like_attention(self): + parallel = fixture.parallel_of(attn_dp=1, attn_tp=2) + linear, sparse = _build_layers(parallel, "attn_tp") + + def steps(stage): + entry = stage.plan.paths[ORDINARY].entry + return entry.prepare.keywords["step"].func, entry.declared_sum + + self.assertEqual( + [steps(stage) for stage in linear], [steps(stage) for stage in sparse] + ) + + def test_token_order_needs_an_always_partial_attention(self): + with self.assertRaises(ValueError): + declare_attn(in_token_order=True, reduction=ProducerReduction.EXIT_SCOPED) + + def test_only_a_token_order_output_is_taken_back(self): + shard, full = Layout(frozenset({TokenAxis.ATTN_CP})), Layout(frozenset()) + moves = CpMoves(gather=prepare._then_moe_cp_gather, take_back=None) + + def edge(produced, need): + return EdgeContract( + produced=produced, need=need, residual=shard, residual_to=shard + ) + + token_order = OutputContract( + full, group=SumGroup.TP, always_partial=True, in_token_order=True + ) + # No model runs two token-order stages back to back. + with self.assertRaises(NotImplementedError): + bind_entry( + edge(token_order, InputContract(full, in_token_order=True)), + cp_moves=moves, + ) + # Rows not in token order are taken back by the CP moves, which here + # cannot complete a TP sum. + rank_major = OutputContract(full, group=SumGroup.TP, always_partial=True) + with self.assertRaises(NotImplementedError): + bind_entry(edge(rank_major, InputContract(shard)), cp_moves=moves) + + def test_the_take_back_sums_then_shards(self): + calls = [] + + def sum_group(group): + def all_reduce(hidden): + calls.append(("sum", group)) + return hidden * 2 + + return SimpleNamespace(all_reduce=all_reduce) + + def shard(hidden, forward_batch): + calls.append(("shard",)) + return hidden[:2] + + residual = torch.ones(2, 3) + with ( + patch.object(ops, "_sum_group", sum_group), + patch.object(ops, "cp_shard_hidden_states", shard), + ): + hidden, kept = cp_take_back_in_token_order( + torch.ones(4, 3), residual, None, group=SumGroup.ATTN_CP + ) + self.assertEqual(calls, [("sum", SumGroup.ATTN_CP), ("shard",)]) + torch.testing.assert_close(hidden, torch.full((2, 3), 2.0)) + self.assertIs(kept, residual) + + +if __name__ == "__main__": + unittest.main()