[Refactor] Decide input-scattered eligibility and stage rejections in one place (#43456)
This commit is contained in:
@@ -23,13 +23,11 @@ import torch
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
attn_tp_all_gather_into_tensor,
|
||||
get_local_dp_buffer,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
input_scattered_configured,
|
||||
)
|
||||
from sglang.srt.layers.moe import get_moe_a2a_backend
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
@@ -81,15 +79,13 @@ class AttnTpContext:
|
||||
# Only MHC pre-gathers hidden states before DSA attention, so non-MHC DSA
|
||||
# cannot use scattered inputs.
|
||||
self.is_dsa = is_dsa
|
||||
# Only narrows input_scattered_configured(), for which stages bind
|
||||
# the input-scattered variant.
|
||||
self.allow_input_scattered = (
|
||||
get_parallel().enable_attn_tp_input_scattered
|
||||
input_scattered_configured()
|
||||
and (_is_cuda or _is_npu)
|
||||
and q_lora_rank is not None
|
||||
and (is_mhc or not is_dsa)
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not is_dense_ffn_fully_dp()
|
||||
and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
and get_spec().speculative_algorithm != "EAGLE3"
|
||||
)
|
||||
|
||||
@@ -17,7 +17,6 @@ from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import Callable, Optional
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.boundary import (
|
||||
ExitMove,
|
||||
@@ -43,19 +42,12 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
TokenAxis,
|
||||
_batch_shards_over_cp,
|
||||
_cp_gathers_over_attn_cp,
|
||||
_prefill_cp_shards_tokens,
|
||||
batches_are_unpadded,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.prepare import (
|
||||
_attn_input_default,
|
||||
_attn_input_scattered,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_forward,
|
||||
get_lora,
|
||||
@@ -64,65 +56,12 @@ from sglang.srt.runtime_context import (
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
|
||||
|
||||
def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
"""A MoE layer under attention CP whose tokens the steps cannot bring
|
||||
to it: under a prefill CP, one dispatched per DP shard under attention DP
|
||||
and GQA CP, and one on the TP group whose data-parallel groups are the CP
|
||||
ranks under DSA or MLA CP; and under attention DP, one on the TP group
|
||||
whose data-parallel groups are the CP ranks."""
|
||||
parallel = get_parallel()
|
||||
gqa = not _cp_gathers_over_attn_cp()
|
||||
if moe_on_local_rows:
|
||||
if cp_shards and gqa and parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
|
||||
)
|
||||
elif parallel.moe_dp_size == parallel.attn_cp_size:
|
||||
if cp_shards and not gqa:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"DSA or MLA prefill CP"
|
||||
)
|
||||
if parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"attention DP and attention CP"
|
||||
)
|
||||
|
||||
|
||||
def _unpadded_possible() -> bool:
|
||||
"""Whether a batch whose rows do not divide over attention TP may reach an
|
||||
FFN that would run on this rank's attention-TP slice, so that FFN needs a
|
||||
variant that stays on the attention's rows."""
|
||||
return batches_are_unpadded() and (
|
||||
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
def _rows_indivisible_over_attn_tp(forward_batch, attn_tp_size: int) -> bool:
|
||||
"""Whether this batch arrived with rows that do not divide over attention TP."""
|
||||
return forward_batch.input_ids.shape[0] % attn_tp_size != 0
|
||||
|
||||
|
||||
def _input_scattered_possible() -> bool:
|
||||
"""Whether a batch may run this layer with input-scattered attention:
|
||||
configured, on TP without attention DP, a prefill CP, an a2a backend or
|
||||
a dense MLP on every rank. The rest of what ``AttnTpContext.init_context`` requires is only
|
||||
known once the model is built."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and parallel.attn_dp_size == 1
|
||||
and not _prefill_cp_shards_tokens()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VariantEdges:
|
||||
"""One stage's incoming and outgoing contracts for one batch variant.
|
||||
@@ -276,19 +215,12 @@ class StagePlan:
|
||||
return BatchVariant.ORDINARY
|
||||
|
||||
def path_for(self, forward_batch):
|
||||
variant = self.variant_for(forward_batch)
|
||||
try:
|
||||
return self.paths[variant]
|
||||
except KeyError:
|
||||
raise NotImplementedError(
|
||||
f"no stage boundary path for the active {variant.name} batch"
|
||||
) from None
|
||||
return _bound_for(self.paths, self.variant_for(forward_batch))
|
||||
|
||||
def fused_input_rows(self, forward_batch):
|
||||
if self._next_input_rows is not None:
|
||||
return self._next_input_rows[self.variant_for(forward_batch)]
|
||||
entry = self.path_for(forward_batch)
|
||||
return entry.entry.input_rows
|
||||
return _bound_for(self._next_input_rows, self.variant_for(forward_batch))
|
||||
return self.path_for(forward_batch).entry.input_rows
|
||||
|
||||
def produced(self, forward_batch):
|
||||
return self.path_for(forward_batch).output
|
||||
@@ -308,6 +240,17 @@ class StagePlan:
|
||||
)
|
||||
|
||||
|
||||
def _bound_for(bound, variant):
|
||||
"""What a stage bound for a batch's variant: the variant a batch selects
|
||||
must be one the stage bound, whichever table is looked up."""
|
||||
try:
|
||||
return bound[variant]
|
||||
except KeyError:
|
||||
raise NotImplementedError(
|
||||
f"no stage boundary path for the active {variant.name} batch"
|
||||
) from None
|
||||
|
||||
|
||||
def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options):
|
||||
if incoming.consumer != declaration or outgoing.producer != declaration:
|
||||
raise ValueError("connections do not match the stage declaration")
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Callable, Mapping, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers import layernorm_sp
|
||||
from sglang.srt.layers.layer_boundary.adapters.overlap import (
|
||||
resolve_exit_rows,
|
||||
@@ -20,10 +21,6 @@ from sglang.srt.layers.layer_boundary.boundary import _cp_moves
|
||||
from sglang.srt.layers.layer_boundary.construction import (
|
||||
BatchVariant,
|
||||
_bind_stage,
|
||||
_input_scattered_possible,
|
||||
_reject_unsupported_cp_moe,
|
||||
_unpadded_possible,
|
||||
_use_ag_after_qlora,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
EdgeContract,
|
||||
@@ -40,6 +37,8 @@ from sglang.srt.layers.layer_boundary.layout import (
|
||||
TokenAxis,
|
||||
_cp_gathers_over_attn_cp,
|
||||
_prefill_cp_shards_tokens,
|
||||
batches_are_unpadded,
|
||||
input_scattered_configured,
|
||||
is_dense_ffn_fully_dp,
|
||||
moe_gathers_over_moe_cp,
|
||||
token_axis_sizes,
|
||||
@@ -58,12 +57,49 @@ from sglang.srt.layers.layer_boundary.stage import StageBoundary
|
||||
from sglang.srt.layers.moe import is_moe_input_scattered_across_dp_ranks
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
|
||||
_use_ag_after_qlora = envs.SGLANG_USE_AG_AFTER_QLORA.get()
|
||||
|
||||
|
||||
def _reject_unsupported_cp_moe(moe_on_local_rows: bool, cp_shards: bool) -> None:
|
||||
"""A MoE layer under attention CP whose tokens the steps cannot bring
|
||||
to it: under a prefill CP, one dispatched per DP shard under attention DP
|
||||
and GQA CP, and one on the TP group whose data-parallel groups are the CP
|
||||
ranks under DSA or MLA CP; and under attention DP, one on the TP group
|
||||
whose data-parallel groups are the CP ranks."""
|
||||
parallel = get_parallel()
|
||||
gqa = not _cp_gathers_over_attn_cp()
|
||||
if moe_on_local_rows:
|
||||
if cp_shards and gqa and parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE dispatched per DP shard under attention DP and GQA prefill CP"
|
||||
)
|
||||
elif parallel.moe_dp_size == parallel.attn_cp_size:
|
||||
if cp_shards and not gqa:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"DSA or MLA prefill CP"
|
||||
)
|
||||
if parallel.attn_dp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"a MoE on the TP group with moe_dp_size == attn_cp_size under "
|
||||
"attention DP and attention CP"
|
||||
)
|
||||
|
||||
|
||||
def _unpadded_possible() -> bool:
|
||||
"""Whether a batch whose rows do not divide over attention TP may reach an
|
||||
FFN that would run on this rank's attention-TP slice, so that FFN needs a
|
||||
variant that stays on the attention's rows."""
|
||||
return batches_are_unpadded() and (
|
||||
is_moe_input_scattered_across_dp_ranks() or is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
def _active_variants():
|
||||
yield BatchVariant.ORDINARY
|
||||
if _prefill_cp_shards_tokens():
|
||||
yield BatchVariant.CONTEXT_PARALLEL
|
||||
if _input_scattered_possible():
|
||||
if input_scattered_configured():
|
||||
yield BatchVariant.INPUT_SCATTERED
|
||||
if layernorm_sp.layernorm_sp_enabled():
|
||||
yield BatchVariant.SEQUENCE_PARALLEL
|
||||
@@ -120,8 +156,6 @@ def _resolve_ffn(
|
||||
cp_shards = _prefill_cp_shards_tokens()
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
on_rank_rows = _ffn_on_rank_rows(sparse, dense_tp_size)
|
||||
if parallel.attn_cp_size > 1 and sparse:
|
||||
_reject_unsupported_cp_moe(on_rank_rows, cp_shards)
|
||||
if variant is BatchVariant.UNPADDED and on_rank_rows:
|
||||
# Rows that do not divide over attention TP stay whole: the FFN runs on
|
||||
# the attention's rows (an a2a MoE dispatches them from every
|
||||
@@ -463,19 +497,32 @@ def declare_ffn(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_stage(stage, variant, following=None):
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
def _reject_unsupported(stage):
|
||||
"""Reject a stage whose declared capabilities this parallel configuration
|
||||
cannot bind, naming the combination. The rejections that depend on the
|
||||
rows of one edge are made where that edge binds (``_bind_stage``)."""
|
||||
parallel = get_parallel()
|
||||
if stage.update.applied_at_exit:
|
||||
if stage.sparse and moe_gathers_over_moe_cp():
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with a MoE gathered over "
|
||||
"the MoE-CP group"
|
||||
)
|
||||
if get_parallel().attn_cp_size > 1 and _input_scattered_possible():
|
||||
if parallel.attn_cp_size > 1 and input_scattered_configured():
|
||||
raise NotImplementedError(
|
||||
"an update applied at the stage's exit with input-scattered "
|
||||
"attention under attention CP"
|
||||
)
|
||||
if stage.kind is StageKind.FFN and stage.sparse and parallel.attn_cp_size > 1:
|
||||
_reject_unsupported_cp_moe(
|
||||
_ffn_on_rank_rows(stage.sparse, stage.dense_tp_size),
|
||||
_prefill_cp_shards_tokens(),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_stage(stage, variant, following=None):
|
||||
axes, attention, local, full = _row_layouts(variant)
|
||||
_reject_unsupported(stage)
|
||||
if stage.kind is StageKind.FFN:
|
||||
declaration, residual, returned = _resolve_ffn(
|
||||
variant,
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.moe import (
|
||||
get_moe_a2a_backend,
|
||||
is_moe_input_scattered_across_dp_ranks,
|
||||
post_experts_reduction_group,
|
||||
)
|
||||
@@ -101,6 +102,23 @@ def _prefill_cp_shards_tokens() -> bool:
|
||||
return parallel.attn_cp_size > 1 and parallel.enable_prefill_cp
|
||||
|
||||
|
||||
def input_scattered_configured() -> bool:
|
||||
"""Whether the parallel configuration allows input-scattered attention:
|
||||
enabled, TP > 1, no attention DP, no prefill CP sharding the tokens, no
|
||||
a2a MoE backend and no dense FFN fully data-parallel. Stages bind the
|
||||
input-scattered variant exactly when this holds; the model's own
|
||||
conditions (``AttnTpContext.init_context``) may only narrow it."""
|
||||
parallel = get_parallel()
|
||||
return (
|
||||
parallel.enable_attn_tp_input_scattered
|
||||
and parallel.tp_size > 1
|
||||
and not parallel.attn_dp_enabled
|
||||
and not _prefill_cp_shards_tokens()
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not is_dense_ffn_fully_dp()
|
||||
)
|
||||
|
||||
|
||||
def _batch_size(forward_batch: ForwardBatch) -> int:
|
||||
return (
|
||||
forward_batch.input_ids.shape[0] if hasattr(forward_batch, "input_ids") else 0
|
||||
|
||||
Reference in New Issue
Block a user