[Refactor] Decide an exit's fixed facts when the stage is built (#43458)
This commit is contained in:
@@ -27,6 +27,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None:
|
||||
tp_size=cfg.tp_size,
|
||||
attn_dp_enabled=attn_dp_enabled_of(cfg),
|
||||
speculative_algorithm=cfg.speculative_algorithm,
|
||||
pp_size=cfg.pp_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -36,6 +37,7 @@ def validate_layernorm_sp(
|
||||
tp_size: int,
|
||||
attn_dp_enabled: bool,
|
||||
speculative_algorithm: Optional[str],
|
||||
pp_size: int = 1,
|
||||
) -> None:
|
||||
"""Fail loud for unsupported / incompatible configs. Callers gate on the flag."""
|
||||
from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES
|
||||
@@ -62,3 +64,9 @@ def validate_layernorm_sp(
|
||||
"(EAGLE/EAGLE3): the captured aux hidden states would be "
|
||||
"sequence-sharded."
|
||||
)
|
||||
if pp_size > 1:
|
||||
raise ValueError(
|
||||
"--enable-layernorm-sp is not compatible with pipeline parallelism: "
|
||||
"only the first rank's stack starts the sequence-sharded region, so "
|
||||
"the next rank would receive a stream it does not expect."
|
||||
)
|
||||
|
||||
@@ -36,7 +36,7 @@ from sglang.srt.layers.layer_boundary.contracts import (
|
||||
StageKind,
|
||||
StagePath,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.exit import ExitPolicy
|
||||
from sglang.srt.layers.layer_boundary.exit import ExitPolicy, exit_facts
|
||||
from sglang.srt.layers.layer_boundary.fusions.allreduce import (
|
||||
attn_input_fusions,
|
||||
ffn_input_fusions,
|
||||
@@ -191,7 +191,7 @@ class StagePlan:
|
||||
attn_tp_gather=exit_gather,
|
||||
)
|
||||
)
|
||||
self.paths[variant] = StagePath(
|
||||
path = StagePath(
|
||||
entry=entry,
|
||||
output=edges.outgoing.produced,
|
||||
output_move=out.output_move,
|
||||
@@ -200,6 +200,9 @@ class StagePlan:
|
||||
returns_over_dp=out.returns_over_dp,
|
||||
writes_at_handoff=writes_at_handoff,
|
||||
)
|
||||
self.paths[variant] = msgspec.structs.replace(
|
||||
path, exit=exit_facts(kind, self, variant, path)
|
||||
)
|
||||
|
||||
@property
|
||||
def incoming_residual_rows(self):
|
||||
|
||||
@@ -262,6 +262,48 @@ class EntryPath(msgspec.Struct, frozen=True):
|
||||
capture_preserves_residual: Optional[Callable] = None
|
||||
|
||||
|
||||
class ExitFacts(msgspec.Struct, frozen=True):
|
||||
"""What an FFN's or a mixer's exit decides from fixed facts, for one batch
|
||||
variant: the parallel configuration, the declarations and the bound path.
|
||||
What depends on the batch (its padding, whether the attention-DP
|
||||
reduce-scatter is usable, whether it has tokens, a fused consumer's
|
||||
accept) stays with the exit's decision for that batch.
|
||||
|
||||
They are read once, when the stage binds, under the scope it is built
|
||||
in: a speculative draft's own MoE backends and boundary reduction while
|
||||
it builds. A worker builds and runs its draft under the same MoE backend
|
||||
scopes, so they hold for every forward. Whether the attention-DP
|
||||
reduce-scatter is usable can change with elastic EP, so it is not one of
|
||||
them.
|
||||
|
||||
Fields:
|
||||
may_defer_sum: Whether an FFN may leave its sum to the next input as
|
||||
far as fixed facts go: TP > 1, not the stack's end, one group the
|
||||
next input can complete it over, no MoE-CP all-gather or
|
||||
input-scattered batch, no EAGLE draft under attention DP, and under
|
||||
attention DP only when the next input also scatters the output back.
|
||||
defers_mixer_sum: Whether a mixer carries its sum to the next attention.
|
||||
sum_in_reduce_scatter: Whether, without an attention-DP reduce-scatter,
|
||||
a reduce-scatter completes the FFN's sum: its exit's own move, or
|
||||
the next input's on an input-scattered batch.
|
||||
sum_left_to_next_input: The sum an input-scattered batch's next input
|
||||
completes, as the stream records it.
|
||||
reduce_scatterv: Whether the attention-DP return is the reduce-scatterv.
|
||||
single_sum: Whether the post-expert sum is one all-reduce, which a
|
||||
deferred sum's consumer completes.
|
||||
fused_consumer_sum: Whether a LoRA or TP1 shared-expert output, which
|
||||
is not one all-reduce, may still be left to a fused consumer.
|
||||
"""
|
||||
|
||||
may_defer_sum: bool = False
|
||||
defers_mixer_sum: bool = False
|
||||
sum_in_reduce_scatter: bool = False
|
||||
sum_left_to_next_input: Optional[SumGroup] = None
|
||||
reduce_scatterv: bool = False
|
||||
single_sum: bool = False
|
||||
fused_consumer_sum: bool = False
|
||||
|
||||
|
||||
class StagePath(msgspec.Struct, frozen=True):
|
||||
"""Precomputed entry and exit work for one stage and batch variant.
|
||||
|
||||
@@ -276,6 +318,7 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
returns_over_dp: Whether output uses batch-dependent attention-DP transport.
|
||||
writes_at_handoff: Whether the exit completes the output and writes it
|
||||
into the residual, for an FFN that hands off to another pipeline rank.
|
||||
exit: What the exit decides from fixed facts (see ExitFacts).
|
||||
"""
|
||||
|
||||
entry: EntryPath
|
||||
@@ -287,6 +330,7 @@ class StagePath(msgspec.Struct, frozen=True):
|
||||
|
||||
returns_over_dp: bool = False
|
||||
writes_at_handoff: bool = False
|
||||
exit: ExitFacts = ExitFacts()
|
||||
|
||||
|
||||
class StageKind(Enum):
|
||||
|
||||
@@ -28,7 +28,11 @@ from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
is_enable_moe_cp_allgather,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context
|
||||
from sglang.srt.layers.layer_boundary.contracts import (
|
||||
BatchVariant,
|
||||
ExitFacts,
|
||||
StageKind,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.layout import (
|
||||
SumGroup,
|
||||
_ffn_has_tokens,
|
||||
@@ -72,13 +76,14 @@ def _select_dp_reduce_scatter(
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
may_reduce_scatter: bool,
|
||||
may_reduce_scatterv: bool,
|
||||
reduce_scatterv: bool,
|
||||
) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]:
|
||||
"""The reduce-scatter that brings an FFN output gathered over attention
|
||||
DP back to this rank's tokens when the FFN leaves its sum to it (see
|
||||
OutputContract); None when the FFN reduces the output and only a scatter
|
||||
remains."""
|
||||
if should_use_dp_reduce_scatterv() and may_reduce_scatterv:
|
||||
remains. Whether the reduce-scatter is usable follows the DP group, which
|
||||
an elastic scale-up changes, so it is read for each batch."""
|
||||
if reduce_scatterv:
|
||||
return dp_reduce_scatterv
|
||||
if (
|
||||
may_reduce_scatter
|
||||
@@ -89,6 +94,98 @@ def _select_dp_reduce_scatter(
|
||||
return None
|
||||
|
||||
|
||||
def exit_facts(kind, plan, variant, path) -> ExitFacts:
|
||||
"""What ``path``'s exit decides from fixed facts on ``variant``: the
|
||||
parallel configuration, the declarations and the bound path. A stage
|
||||
without an exit (an attention that always leaves its sum) decides
|
||||
nothing. A condition is read only where the exit used to read it."""
|
||||
if plan.finishes_directly:
|
||||
return ExitFacts()
|
||||
output = path.output
|
||||
scattered = variant is BatchVariant.INPUT_SCATTERED
|
||||
tp = get_parallel().tp_size > 1
|
||||
if kind is StageKind.ATTENTION:
|
||||
return ExitFacts(
|
||||
defers_mixer_sum=output.may_defer_to_next
|
||||
and tp
|
||||
and not is_dp_attention_enabled()
|
||||
and _sum_deferral_allowed(plan, output, scattered=scattered)
|
||||
)
|
||||
sum_in_reduce_scatter = output.may_reduce_scatter and (
|
||||
path.output_move_completes_sum or (scattered and not plan.terminal)
|
||||
)
|
||||
may_defer_sum = (
|
||||
tp
|
||||
and not plan.terminal
|
||||
and _sum_deferral_allowed(plan, output, scattered=scattered)
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
and (
|
||||
not is_dp_attention_enabled()
|
||||
or (path.returns_over_dp and not output.update.applied_at_exit)
|
||||
)
|
||||
)
|
||||
single_sum = may_defer_sum and post_experts_sum_is_one_all_reduce()
|
||||
return ExitFacts(
|
||||
may_defer_sum=may_defer_sum,
|
||||
sum_in_reduce_scatter=sum_in_reduce_scatter,
|
||||
# On an input-scattered batch the next input's TP reduce-scatter
|
||||
# completes the sum; the other reduce-scatters are this exit's move.
|
||||
sum_left_to_next_input=(
|
||||
SumGroup.TP
|
||||
if sum_in_reduce_scatter
|
||||
and not path.returns_over_dp
|
||||
and not path.output_move_completes_sum
|
||||
else None
|
||||
),
|
||||
reduce_scatterv=path.returns_over_dp
|
||||
and should_use_dp_reduce_scatterv()
|
||||
and output.may_reduce_scatterv,
|
||||
single_sum=single_sum,
|
||||
fused_consumer_sum=may_defer_sum and not single_sum and _fused_consumer_sum(),
|
||||
)
|
||||
|
||||
|
||||
def _sum_deferral_allowed(plan, output, *, scattered: bool) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip complete_now
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if is_enable_moe_cp_allgather() or output.group is None:
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
# two disjoint groups; post_experts_all_reduce() merges them into one
|
||||
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
|
||||
# When merging is blocked, no single group covers both, so fusion stays off.
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.moe_ep_size > 1
|
||||
and parallel.moe_tp_size > 1
|
||||
and not can_merge_post_experts_all_reduce()
|
||||
):
|
||||
return False
|
||||
|
||||
if (
|
||||
is_dp_attention_enabled()
|
||||
and plan._speculative_algo is not None
|
||||
and plan._speculative_algo.is_eagle()
|
||||
):
|
||||
return False
|
||||
|
||||
return not scattered
|
||||
|
||||
|
||||
def _fused_consumer_sum() -> bool:
|
||||
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
|
||||
# when the sum is deferred. Preserve these fused paths, but keep their
|
||||
# producer-side reduction order when no fused consumer is enabled.
|
||||
return (
|
||||
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
and post_experts_reduction_group() is get_parallel().tp_group
|
||||
)
|
||||
|
||||
|
||||
class ExitPolicy:
|
||||
"""Choose and finish one producer's output; the batch owns its residual."""
|
||||
|
||||
@@ -133,18 +230,6 @@ class ExitPolicy:
|
||||
return stream.write(hidden_states)
|
||||
return stream.record(hidden_states, update, declared_sum=declared_sum)
|
||||
|
||||
@staticmethod
|
||||
def _sum_left_to_next_input(steps, sum_in_reduce_scatter):
|
||||
# On an input-scattered batch the next input's TP reduce-scatter
|
||||
# completes the sum; the other reduce-scatters are this exit's move.
|
||||
if (
|
||||
sum_in_reduce_scatter
|
||||
and not steps.returns_over_dp
|
||||
and not steps.output_move_completes_sum
|
||||
):
|
||||
return SumGroup.TP
|
||||
return None
|
||||
|
||||
def _next_input_can_scatter(self, steps) -> bool:
|
||||
"""Whether the next layer's input can run this layer's move of its FFN
|
||||
output back to this rank's tokens: the base postprocess scatter, when
|
||||
@@ -162,19 +247,13 @@ class ExitPolicy:
|
||||
return _select_dp_reduce_scatter(
|
||||
forward_batch,
|
||||
may_reduce_scatter=steps.output.may_reduce_scatter,
|
||||
may_reduce_scatterv=steps.output.may_reduce_scatterv,
|
||||
reduce_scatterv=steps.exit.reduce_scatterv,
|
||||
)
|
||||
|
||||
def _sum_in_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool:
|
||||
"""Whether a reduce-scatter completes the FFN output's sum: the
|
||||
attention-DP one ``dp_step`` names, or the CP / input-scattered one."""
|
||||
if dp_step is not None:
|
||||
return True
|
||||
if not steps.output.may_reduce_scatter:
|
||||
return False
|
||||
if steps.output_move_completes_sum:
|
||||
return True
|
||||
return get_attn_tp_context().input_scattered and not self.plan.terminal
|
||||
return dp_step is not None or steps.exit.sum_in_reduce_scatter
|
||||
|
||||
def ffn_reduction_group(self, steps) -> GroupCoordinator:
|
||||
"""The group this layer's FFN output owes its sum over: the MoE output's
|
||||
@@ -307,33 +386,6 @@ class ExitPolicy:
|
||||
off its MoE finalize, then call ``finish``."""
|
||||
return FfnExit(self, forward_batch, stream=stream)
|
||||
|
||||
def _sum_deferral_allowed(self, steps) -> bool:
|
||||
# Under the MoE-CP all-gather the fusion path would skip complete_now
|
||||
# and its MoE-CP scatter, leaving hidden_states longer than the residual.
|
||||
if is_enable_moe_cp_allgather() or steps.output.group is None:
|
||||
return False
|
||||
|
||||
# The fused residual+LN reduces over a single group. Hybrid EP+TP spans
|
||||
# two disjoint groups; post_experts_all_reduce() merges them into one
|
||||
# _TP reduction when moe_dp_size == 1, which the fused kernel can absorb.
|
||||
# When merging is blocked, no single group covers both, so fusion stays off.
|
||||
parallel = get_parallel()
|
||||
if (
|
||||
parallel.moe_ep_size > 1
|
||||
and parallel.moe_tp_size > 1
|
||||
and not can_merge_post_experts_all_reduce()
|
||||
):
|
||||
return False
|
||||
|
||||
if (
|
||||
is_dp_attention_enabled()
|
||||
and self.plan._speculative_algo is not None
|
||||
and self.plan._speculative_algo.is_eagle()
|
||||
):
|
||||
return False
|
||||
|
||||
return not get_attn_tp_context().input_scattered
|
||||
|
||||
def _defers_sum(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -344,23 +396,20 @@ class ExitPolicy:
|
||||
) -> bool:
|
||||
"""Whether the FFN leaves its output's all-reduce to the next layer's
|
||||
input when no fused kernel takes it: when the next layer would run the
|
||||
same all-reduce the FFN itself would have."""
|
||||
same all-reduce the FFN itself would have. A reduce-scatter completing
|
||||
the sum rules it out, and under attention DP so does any move back but
|
||||
the scatter the next input runs."""
|
||||
return (
|
||||
get_parallel().tp_size > 1
|
||||
and not self.plan.terminal
|
||||
and self._sum_deferral_allowed(steps)
|
||||
and _batch_allows_deferred_sum(forward_batch, self.plan)
|
||||
steps.exit.may_defer_sum
|
||||
and _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan)
|
||||
and not sum_in_reduce_scatter
|
||||
# Under attention DP the next layer must also run postprocess's
|
||||
# scatter back to this rank's tokens, and nothing more.
|
||||
and (
|
||||
not is_dp_attention_enabled()
|
||||
or (self._next_input_can_scatter(steps) and dp_step is None)
|
||||
)
|
||||
and dp_step is None
|
||||
)
|
||||
|
||||
|
||||
def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bool:
|
||||
def _batch_allows_deferred_sum(
|
||||
forward_batch: ForwardBatch, facts: ExitFacts, boundary=None
|
||||
) -> bool:
|
||||
"""Admit ordinary single-sum outputs, plus LoRA and TP1 shared-expert
|
||||
outputs when a fused consumer (the backend's can_defer_all_reduce,
|
||||
FlashInfer or aiter) can take them.
|
||||
@@ -370,17 +419,9 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo
|
||||
"""
|
||||
if not _ffn_has_tokens(forward_batch):
|
||||
return False
|
||||
if post_experts_sum_is_one_all_reduce():
|
||||
if facts.single_sum:
|
||||
return True
|
||||
# LoRA-B is replicated and linear. TP1 shared experts add on rank zero
|
||||
# when the sum is deferred. Preserve these fused paths, but keep their
|
||||
# producer-side reduction order when no fused consumer is enabled.
|
||||
if not (
|
||||
(get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get())
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and not get_exec().comm.enable_quant_communications
|
||||
and post_experts_reduction_group() is get_parallel().tp_group
|
||||
):
|
||||
if not facts.fused_consumer_sum:
|
||||
return False
|
||||
if (
|
||||
boundary is not None
|
||||
@@ -452,12 +493,7 @@ class MixerExit:
|
||||
self._update = produced.update
|
||||
self._group = produced.group
|
||||
self._declared_sum = produced.group if produced.always_partial else None
|
||||
self._defers = (
|
||||
produced.may_defer_to_next
|
||||
and get_parallel().tp_size > 1
|
||||
and not is_dp_attention_enabled()
|
||||
and boundary._sum_deferral_allowed(steps)
|
||||
)
|
||||
self._defers = steps.exit.defers_mixer_sum
|
||||
|
||||
def __enter__(self) -> MixerExit:
|
||||
return self
|
||||
@@ -519,14 +555,16 @@ class FfnExit:
|
||||
):
|
||||
self._stream = stream
|
||||
self.boundary = boundary
|
||||
# The same path the entry took: the flags that select it hold for the
|
||||
# whole call, and a stage keeps no state between its entry and exit.
|
||||
steps = boundary.plan.path_for(forward_batch)
|
||||
completion = boundary._decide(forward_batch, steps)
|
||||
self.defer_moe_finalize = completion.defer_moe_finalize
|
||||
self._complete = completion.complete
|
||||
self._update = steps.output.update
|
||||
self._declared_sum = boundary._sum_left_to_next_input(
|
||||
steps, completion.sum_in_reduce_scatter
|
||||
)
|
||||
# A reduce-scatter that completes the sum on the way back is the
|
||||
# next input's only on an input-scattered batch.
|
||||
self._declared_sum = steps.exit.sum_left_to_next_input
|
||||
self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize)
|
||||
|
||||
def __enter__(self) -> FfnExit:
|
||||
|
||||
@@ -317,6 +317,13 @@ class StageBoundary:
|
||||
return hidden_states
|
||||
|
||||
def _prepare_input(self, hidden_states, stream, forward_batch, **call):
|
||||
hidden_states, steps = self._select(hidden_states, forward_batch)
|
||||
return self._prepare(hidden_states, stream, forward_batch, steps, **call)
|
||||
|
||||
def _select(self, hidden_states, forward_batch):
|
||||
"""This call's path, looked up once the flags that select it are set:
|
||||
the stack's first stage starts a sequence-parallel batch. Publishes
|
||||
the rows LoRA reads for the stage's input."""
|
||||
plan = self.plan
|
||||
if self.kind is StageKind.ATTENTION:
|
||||
publish_attn_lora_rows(plan._publish_lora_layout)
|
||||
@@ -329,11 +336,10 @@ class StageBoundary:
|
||||
)
|
||||
if get_forward().sp_active:
|
||||
hidden_states = layernorm_sp.sp_entry_scatter(hidden_states)
|
||||
else:
|
||||
publish_ffn_lora_rows(
|
||||
plan._publish_lora_layout, self.entry(forward_batch).input_rows
|
||||
)
|
||||
return self._prepare(hidden_states, stream, forward_batch, **call)
|
||||
steps = plan.path_for(forward_batch)
|
||||
if self.kind is StageKind.FFN:
|
||||
publish_ffn_lora_rows(plan._publish_lora_layout, steps.entry.input_rows)
|
||||
return hidden_states, steps
|
||||
|
||||
def _prepare_attention(
|
||||
self,
|
||||
@@ -362,14 +368,16 @@ class StageBoundary:
|
||||
capture(value)
|
||||
else:
|
||||
capture(value + previous, owned=True)
|
||||
hidden_states, stream = self._prepare_input(
|
||||
hidden_states, steps = self._select(hidden_states, forward_batch)
|
||||
hidden_states, stream = self._prepare(
|
||||
hidden_states,
|
||||
stream,
|
||||
forward_batch,
|
||||
steps,
|
||||
quant_format=quant_format,
|
||||
post_residual_addition=post_residual_addition,
|
||||
)
|
||||
entry = self.entry(forward_batch)
|
||||
entry = steps.entry
|
||||
if capture is not None and not capture_before_read:
|
||||
value = stream.residual
|
||||
move = entry.capture_move
|
||||
|
||||
@@ -98,6 +98,7 @@ def stub_plan():
|
||||
plan._unpadded_attn_tp_size = None
|
||||
plan.paths = {}
|
||||
plan.enters_stack = False
|
||||
plan.finishes_directly = False
|
||||
return plan
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user