[Refactor] Decide input-scattered eligibility and stage rejections in one place (#43456)

This commit is contained in:
Cheng Wan
2026-10-10 12:41:38 -07:00
committed by GitHub
parent b0dde87de2
commit d2c597c8db
5 changed files with 94 additions and 90 deletions
@@ -23,13 +23,11 @@ import torch
from sglang.srt.layers.dp_attention import (
attn_tp_all_gather_into_tensor,
get_local_dp_buffer,
is_dp_attention_enabled,
)
from sglang.srt.layers.layer_boundary.layout import (
batches_are_unpadded,
is_dense_ffn_fully_dp,
input_scattered_configured,
)
from sglang.srt.layers.moe import get_moe_a2a_backend
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
@@ -81,15 +79,13 @@ class AttnTpContext:
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
# cannot use scattered inputs.
self.is_dsa = is_dsa
# Only narrows input_scattered_configured(), for which stages bind
# the input-scattered variant.
self.allow_input_scattered = (
get_parallel().enable_attn_tp_input_scattered
input_scattered_configured()
and (_is_cuda or _is_npu)
and q_lora_rank is not None
and (is_mhc or not is_dsa)
and get_parallel().tp_size > 1
and not is_dp_attention_enabled()
and get_moe_a2a_backend().is_none()
and not is_dense_ffn_fully_dp()
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
and get_spec().speculative_algorithm != "EAGLE3"
)
@@ -17,7 +17,6 @@ from dataclasses import dataclass
from functools import cached_property
from typing import Callable, Optional
from sglang.srt.environ import envs
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
from sglang.srt.layers.layer_boundary.boundary import (
ExitMove,
@@ -43,19 +42,12 @@ from sglang.srt.layers.layer_boundary.layout import (
TokenAxis,
_batch_shards_over_cp,
_cp_gathers_over_attn_cp,
_prefill_cp_shards_tokens,
batches_are_unpadded,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.prepare import (
_attn_input_default,
_attn_input_scattered,
)
from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
is_moe_input_scattered_across_dp_ranks,
)
from sglang.srt.runtime_context import (
get_forward,
get_lora,
@@ -64,65 +56,12 @@ from sglang.srt.runtime_context import (
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
"""A MoE layer under attention CP whose tokens the steps cannot bring
to it: under a prefill CP, one dispatched per DP shard under attention DP
and GQA CP, and one on the TP group whose data-parallel groups are the CP
ranks under DSA or MLA CP; and under attention DP, one on the TP group
whose data-parallel groups are the CP ranks."""
parallel = get_parallel()
gqa = not _cp_gathers_over_attn_cp()
if moe_on_local_rows:
if cp_shards and gqa and parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
)
elif parallel.moe_dp_size == parallel.attn_cp_size:
if cp_shards and not gqa:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"DSA or MLA prefill CP"
)
if parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"attention DP and attention CP"
)
def _unpadded_possible() -> bool:
"""Whether a batch whose rows do not divide over attention TP may reach an
FFN that would run on this rank's attention-TP slice, so that FFN needs a
variant that stays on the attention's rows."""
return batches_are_unpadded() and (
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
)
def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool:
"""Whether this batch arrived with rows that do not divide over attention TP."""
return forward_batch.input_ids.shape[0] % attn_tp_size != 0
def _input_scattered_possible() -> bool:
"""Whether a batch may run this layer with input-scattered attention:
configured, on TP without attention DP, a prefill CP, an a2a backend or
a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
known once the model is built."""
parallel = get_parallel()
return (
parallel.enable_attn_tp_input_scattered
and parallel.tp_size > 1
and parallel.attn_dp_size == 1
and not _prefill_cp_shards_tokens()
and get_moe_a2a_backend().is_none()
and not is_dense_ffn_fully_dp()
)
@dataclass(frozen=True)
class VariantEdges:
"""One stage's incoming and outgoing contracts for one batch variant.
@@ -276,19 +215,12 @@ class StagePlan:
return BatchVariant.ORDINARY
def path_for(self, forward_batch):
variant = self.variant_for(forward_batch)
try:
return self.paths[variant]
except KeyError:
raise NotImplementedError(
f"no stage boundary path for the active {variant.name} batch"
) from None
return _bound_for(self.paths, self.variant_for(forward_batch))
def fused_input_rows(self, forward_batch):
if self._next_input_rows is not None:
return self._next_input_rows[self.variant_for(forward_batch)]
entry = self.path_for(forward_batch)
return entry.entry.input_rows
return _bound_for(self._next_input_rows, self.variant_for(forward_batch))
return self.path_for(forward_batch).entry.input_rows
def produced(self, forward_batch):
return self.path_for(forward_batch).output
@@ -308,6 +240,17 @@ class StagePlan:
)
def _bound_for(bound, variant):
"""What a stage bound for a batch's variant: the variant a batch selects
must be one the stage bound, whichever table is looked up."""
try:
return bound[variant]
except KeyError:
raise NotImplementedError(
f"no stage boundary path for the active {variant.name} batch"
) from None
def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options):
if incoming.consumer != declaration or outgoing.producer != declaration:
raise ValueError("connections do not match the stage declaration")
@@ -11,6 +11,7 @@ from typing import Callable, Mapping, Optional
import msgspec
from sglang.srt.environ import envs
from sglang.srt.layers import layernorm_sp
from sglang.srt.layers.layer_boundary.adapters.overlap import (
resolve_exit_rows,
@@ -20,10 +21,6 @@ from sglang.srt.layers.layer_boundary.boundary import _cp_moves
from sglang.srt.layers.layer_boundary.construction import (
BatchVariant,
_bind_stage,
_input_scattered_possible,
_reject_unsupported_cp_moe,
_unpadded_possible,
_use_ag_after_qlora,
)
from sglang.srt.layers.layer_boundary.contracts import (
EdgeContract,
@@ -40,6 +37,8 @@ from sglang.srt.layers.layer_boundary.layout import (
TokenAxis,
_cp_gathers_over_attn_cp,
_prefill_cp_shards_tokens,
batches_are_unpadded,
input_scattered_configured,
is_dense_ffn_fully_dp,
moe_gathers_over_moe_cp,
token_axis_sizes,
@@ -58,12 +57,49 @@ from sglang.srt.layers.layer_boundary.stage import StageBoundary
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
from sglang.srt.runtime_context import get_exec, get_parallel
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
"""A MoE layer under attention CP whose tokens the steps cannot bring
to it: under a prefill CP, one dispatched per DP shard under attention DP
and GQA CP, and one on the TP group whose data-parallel groups are the CP
ranks under DSA or MLA CP; and under attention DP, one on the TP group
whose data-parallel groups are the CP ranks."""
parallel = get_parallel()
gqa = not _cp_gathers_over_attn_cp()
if moe_on_local_rows:
if cp_shards and gqa and parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
)
elif parallel.moe_dp_size == parallel.attn_cp_size:
if cp_shards and not gqa:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"DSA or MLA prefill CP"
)
if parallel.attn_dp_size > 1:
raise NotImplementedError(
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
"attention DP and attention CP"
)
def _unpadded_possible() -> bool:
"""Whether a batch whose rows do not divide over attention TP may reach an
FFN that would run on this rank's attention-TP slice, so that FFN needs a
variant that stays on the attention's rows."""
return batches_are_unpadded() and (
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
)
def _active_variants():
yield BatchVariant.ORDINARY
if _prefill_cp_shards_tokens():
yield BatchVariant.CONTEXT_PARALLEL
if _input_scattered_possible():
if input_scattered_configured():
yield BatchVariant.INPUT_SCATTERED
if layernorm_sp.layernorm_sp_enabled():
yield BatchVariant.SEQUENCE_PARALLEL
@@ -120,8 +156,6 @@ def _resolve_ffn(
cp_shards = _prefill_cp_shards_tokens()
axes, attention, local, full = _row_layouts(variant)
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
if parallel.attn_cp_size > 1 and sparse:
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
if variant is BatchVariant.UNPADDED and on_rank_rows:
# Rows that do not divide over attention TP stay whole: the FFN runs on
# the attention's rows (an a2a MoE dispatches them from every
@@ -463,19 +497,32 @@ def declare_ffn(
)
def _resolve_stage(stage, variant, following=None):
axes, attention, local, full = _row_layouts(variant)
def _reject_unsupported(stage):
"""Reject a stage whose declared capabilities this parallel configuration
cannot bind, naming the combination. The rejections that depend on the
rows of one edge are made where that edge binds (``_bind_stage``)."""
parallel = get_parallel()
if stage.update.applied_at_exit:
if stage.sparse and moe_gathers_over_moe_cp():
raise NotImplementedError(
"an update applied at the stage's exit with a MoE gathered over "
"the MoE-CP group"
)
if get_parallel().attn_cp_size > 1 and _input_scattered_possible():
if parallel.attn_cp_size > 1 and input_scattered_configured():
raise NotImplementedError(
"an update applied at the stage's exit with input-scattered "
"attention under attention CP"
)
if stage.kind is StageKind.FFN and stage.sparse and parallel.attn_cp_size > 1:
_reject_unsupported_cp_moe(
_ffn_on_rank_rows(stage.sparse, stage.dense_tp_size),
_prefill_cp_shards_tokens(),
)
def _resolve_stage(stage, variant, following=None):
axes, attention, local, full = _row_layouts(variant)
_reject_unsupported(stage)
if stage.kind is StageKind.FFN:
declaration, residual, returned = _resolve_ffn(
variant,
@@ -31,6 +31,7 @@ from sglang.srt.layers.dp_attention import (
is_enable_moe_cp_allgather,
)
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
is_moe_input_scattered_across_dp_ranks,
post_experts_reduction_group,
)
@@ -101,6 +102,23 @@ def _prefill_cp_shards_tokens() -> bool:
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
def input_scattered_configured() -> bool:
"""Whether the parallel configuration allows input-scattered attention:
enabled, TP > 1, no attention DP, no prefill CP sharding the tokens, no
a2a MoE backend and no dense FFN fully data-parallel. Stages bind the
input-scattered variant exactly when this holds; the model's own
conditions (``AttnTpContext.init_context``) may only narrow it."""
parallel = get_parallel()
return (
parallel.enable_attn_tp_input_scattered
and parallel.tp_size > 1
and not parallel.attn_dp_enabled
and not _prefill_cp_shards_tokens()
and get_moe_a2a_backend().is_none()
and not is_dense_ffn_fully_dp()
)
def _batch_size(forward_batch: ForwardBatch) -> int:
return (
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0