diff --git a/python/sglang/srt/layers/layer_boundary/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py index 5f504dc31d..b84dcafa58 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -23,13 +23,11 @@ import torch from sglang.srt.layers.dp_attention import ( attn_tp_all_gather_into_tensor, get_local_dp_buffer, - is_dp_attention_enabled, ) from sglang.srt.layers.layer_boundary.layout import ( batches_are_unpadded, - is_dense_ffn_fully_dp, + input_scattered_configured, ) -from sglang.srt.layers.moe import get_moe_a2a_backend from sglang.srt.model_executor.cuda_graph_config import ( Backend, Phase, @@ -81,15 +79,13 @@ class AttnTpContext: # Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA # cannot use scattered inputs. self.is_dsa = is_dsa + # Only narrows input_scattered_configured(), for which stages bind + # the input-scattered variant. self.allow_input_scattered = ( - get_parallel().enable_attn_tp_input_scattered + input_scattered_configured() and (_is_cuda or _is_npu) and q_lora_rank is not None and (is_mhc or not is_dsa) - and get_parallel().tp_size > 1 - and not is_dp_attention_enabled() - and get_moe_a2a_backend().is_none() - and not is_dense_ffn_fully_dp() and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) and get_spec().speculative_algorithm != "EAGLE3" ) diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index ca400e278c..679516d59a 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -17,7 +17,6 @@ from dataclasses import dataclass from functools import cached_property from typing import Callable, Optional -from sglang.srt.environ import envs from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context from sglang.srt.layers.layer_boundary.boundary import ( ExitMove, @@ -43,19 +42,12 @@ from sglang.srt.layers.layer_boundary.layout import ( TokenAxis, _batch_shards_over_cp, _cp_gathers_over_attn_cp, - _prefill_cp_shards_tokens, - batches_are_unpadded, - is_dense_ffn_fully_dp, ) from sglang.srt.layers.layer_boundary.prepare import ( _attn_input_default, _attn_input_scattered, ) from sglang.srt.layers.layer_boundary.stage import StageBoundary -from sglang.srt.layers.moe import ( - get_moe_a2a_backend, - is_moe_input_scattered_across_dp_ranks, -) from sglang.srt.runtime_context import ( get_forward, get_lora, @@ -64,65 +56,12 @@ from sglang.srt.runtime_context import ( ) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm -_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() - - -def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: - """A MoE layer under attention CP whose tokens the steps cannot bring - to it: under a prefill CP, one dispatched per DP shard under attention DP - and GQA CP, and one on the TP group whose data-parallel groups are the CP - ranks under DSA or MLA CP; and under attention DP, one on the TP group - whose data-parallel groups are the CP ranks.""" - parallel = get_parallel() - gqa = not _cp_gathers_over_attn_cp() - if moe_on_local_rows: - if cp_shards and gqa and parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE dispatched per DP shard under attention DP and GQA prefill CP" - ) - elif parallel.moe_dp_size == parallel.attn_cp_size: - if cp_shards and not gqa: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "DSA or MLA prefill CP" - ) - if parallel.attn_dp_size > 1: - raise NotImplementedError( - "a MoE on the TP group with moe_dp_size == attn_cp_size under " - "attention DP and attention CP" - ) - - -def _unpadded_possible() -> bool: - """Whether a batch whose rows do not divide over attention TP may reach an - FFN that would run on this rank's attention-TP slice, so that FFN needs a - variant that stays on the attention's rows.""" - return batches_are_unpadded() and ( - is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp() - ) - def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool: """Whether this batch arrived with rows that do not divide over attention TP.""" return forward_batch.input_ids.shape[0] % attn_tp_size != 0 -def _input_scattered_possible() -> bool: - """Whether a batch may run this layer with input-scattered attention: - configured, on TP without attention DP, a prefill CP, an a2a backend or - a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only - known once the model is built.""" - parallel = get_parallel() - return ( - parallel.enable_attn_tp_input_scattered - and parallel.tp_size > 1 - and parallel.attn_dp_size == 1 - and not _prefill_cp_shards_tokens() - and get_moe_a2a_backend().is_none() - and not is_dense_ffn_fully_dp() - ) - - @dataclass(frozen=True) class VariantEdges: """One stage's incoming and outgoing contracts for one batch variant. @@ -276,19 +215,12 @@ class StagePlan: return BatchVariant.ORDINARY def path_for(self, forward_batch): - variant = self.variant_for(forward_batch) - try: - return self.paths[variant] - except KeyError: - raise NotImplementedError( - f"no stage boundary path for the active {variant.name} batch" - ) from None + return _bound_for(self.paths, self.variant_for(forward_batch)) def fused_input_rows(self, forward_batch): if self._next_input_rows is not None: - return self._next_input_rows[self.variant_for(forward_batch)] - entry = self.path_for(forward_batch) - return entry.entry.input_rows + return _bound_for(self._next_input_rows, self.variant_for(forward_batch)) + return self.path_for(forward_batch).entry.input_rows def produced(self, forward_batch): return self.path_for(forward_batch).output @@ -308,6 +240,17 @@ class StagePlan: ) +def _bound_for(bound, variant): + """What a stage bound for a batch's variant: the variant a batch selects + must be one the stage bound, whichever table is looked up.""" + try: + return bound[variant] + except KeyError: + raise NotImplementedError( + f"no stage boundary path for the active {variant.name} batch" + ) from None + + def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options): if incoming.consumer != declaration or outgoing.producer != declaration: raise ValueError("connections do not match the stage declaration") diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index f576ce481b..ddf460f86e 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -11,6 +11,7 @@ from typing import Callable, Mapping, Optional import msgspec +from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp from sglang.srt.layers.layer_boundary.adapters.overlap import ( resolve_exit_rows, @@ -20,10 +21,6 @@ from sglang.srt.layers.layer_boundary.boundary import _cp_moves from sglang.srt.layers.layer_boundary.construction import ( BatchVariant, _bind_stage, - _input_scattered_possible, - _reject_unsupported_cp_moe, - _unpadded_possible, - _use_ag_after_qlora, ) from sglang.srt.layers.layer_boundary.contracts import ( EdgeContract, @@ -40,6 +37,8 @@ from sglang.srt.layers.layer_boundary.layout import ( TokenAxis, _cp_gathers_over_attn_cp, _prefill_cp_shards_tokens, + batches_are_unpadded, + input_scattered_configured, is_dense_ffn_fully_dp, moe_gathers_over_moe_cp, token_axis_sizes, @@ -58,12 +57,49 @@ from sglang.srt.layers.layer_boundary.stage import StageBoundary from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks from sglang.srt.runtime_context import get_exec, get_parallel +_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get() + + +def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None: + """A MoE layer under attention CP whose tokens the steps cannot bring + to it: under a prefill CP, one dispatched per DP shard under attention DP + and GQA CP, and one on the TP group whose data-parallel groups are the CP + ranks under DSA or MLA CP; and under attention DP, one on the TP group + whose data-parallel groups are the CP ranks.""" + parallel = get_parallel() + gqa = not _cp_gathers_over_attn_cp() + if moe_on_local_rows: + if cp_shards and gqa and parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE dispatched per DP shard under attention DP and GQA prefill CP" + ) + elif parallel.moe_dp_size == parallel.attn_cp_size: + if cp_shards and not gqa: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "DSA or MLA prefill CP" + ) + if parallel.attn_dp_size > 1: + raise NotImplementedError( + "a MoE on the TP group with moe_dp_size == attn_cp_size under " + "attention DP and attention CP" + ) + + +def _unpadded_possible() -> bool: + """Whether a batch whose rows do not divide over attention TP may reach an + FFN that would run on this rank's attention-TP slice, so that FFN needs a + variant that stays on the attention's rows.""" + return batches_are_unpadded() and ( + is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp() + ) + def _active_variants(): yield BatchVariant.ORDINARY if _prefill_cp_shards_tokens(): yield BatchVariant.CONTEXT_PARALLEL - if _input_scattered_possible(): + if input_scattered_configured(): yield BatchVariant.INPUT_SCATTERED if layernorm_sp.layernorm_sp_enabled(): yield BatchVariant.SEQUENCE_PARALLEL @@ -120,8 +156,6 @@ def _resolve_ffn( cp_shards = _prefill_cp_shards_tokens() axes, attention, local, full = _row_layouts(variant) on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size) - if parallel.attn_cp_size > 1 and sparse: - _reject_unsupported_cp_moe(on_rank_rows, cp_shards) if variant is BatchVariant.UNPADDED and on_rank_rows: # Rows that do not divide over attention TP stay whole: the FFN runs on # the attention's rows (an a2a MoE dispatches them from every @@ -463,19 +497,32 @@ def declare_ffn( ) -def _resolve_stage(stage, variant, following=None): - axes, attention, local, full = _row_layouts(variant) +def _reject_unsupported(stage): + """Reject a stage whose declared capabilities this parallel configuration + cannot bind, naming the combination. The rejections that depend on the + rows of one edge are made where that edge binds (``_bind_stage``).""" + parallel = get_parallel() if stage.update.applied_at_exit: if stage.sparse and moe_gathers_over_moe_cp(): raise NotImplementedError( "an update applied at the stage's exit with a MoE gathered over " "the MoE-CP group" ) - if get_parallel().attn_cp_size > 1 and _input_scattered_possible(): + if parallel.attn_cp_size > 1 and input_scattered_configured(): raise NotImplementedError( "an update applied at the stage's exit with input-scattered " "attention under attention CP" ) + if stage.kind is StageKind.FFN and stage.sparse and parallel.attn_cp_size > 1: + _reject_unsupported_cp_moe( + _ffn_on_rank_rows(stage.sparse, stage.dense_tp_size), + _prefill_cp_shards_tokens(), + ) + + +def _resolve_stage(stage, variant, following=None): + axes, attention, local, full = _row_layouts(variant) + _reject_unsupported(stage) if stage.kind is StageKind.FFN: declaration, residual, returned = _resolve_ffn( variant, diff --git a/python/sglang/srt/layers/layer_boundary/layout.py b/python/sglang/srt/layers/layer_boundary/layout.py index 32816cadc2..04204802f2 100644 --- a/python/sglang/srt/layers/layer_boundary/layout.py +++ b/python/sglang/srt/layers/layer_boundary/layout.py @@ -31,6 +31,7 @@ from sglang.srt.layers.dp_attention import ( is_enable_moe_cp_allgather, ) from sglang.srt.layers.moe import ( + get_moe_a2a_backend, is_moe_input_scattered_across_dp_ranks, post_experts_reduction_group, ) @@ -101,6 +102,23 @@ def _prefill_cp_shards_tokens() -> bool: return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp +def input_scattered_configured() -> bool: + """Whether the parallel configuration allows input-scattered attention: + enabled, TP > 1, no attention DP, no prefill CP sharding the tokens, no + a2a MoE backend and no dense FFN fully data-parallel. Stages bind the + input-scattered variant exactly when this holds; the model's own + conditions (``AttnTpContext.init_context``) may only narrow it.""" + parallel = get_parallel() + return ( + parallel.enable_attn_tp_input_scattered + and parallel.tp_size > 1 + and not parallel.attn_dp_enabled + and not _prefill_cp_shards_tokens() + and get_moe_a2a_backend().is_none() + and not is_dense_ffn_fully_dp() + ) + + def _batch_size(forward_batch: ForwardBatch) -> int: return ( forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0 diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 8b22b90f22..c0b7d84677 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -1709,7 +1709,7 @@ def running(*, reduce_scatterv, a2a=False, use_reduce_scatter=True): # What the MoE declares its skipped reduction leaves, read in its module. for name, value in ( ("get_parallel", lambda: state().parallel), - ("get_moe_a2a_backend", comm_layer.get_moe_a2a_backend), + ("get_moe_a2a_backend", comm_layout.get_moe_a2a_backend), ( "post_experts_output_is_complete", replaced["post_experts_output_is_complete"],