diff --git a/python/sglang/srt/arg_groups/layernorm_sp_hook.py b/python/sglang/srt/arg_groups/layernorm_sp_hook.py index 025a642c80..9fc2b0ade4 100644 --- a/python/sglang/srt/arg_groups/layernorm_sp_hook.py +++ b/python/sglang/srt/arg_groups/layernorm_sp_hook.py @@ -27,6 +27,7 @@ def handle_layernorm_sp(server_args: ServerArgs) -> None: tp_size=cfg.tp_size, attn_dp_enabled=attn_dp_enabled_of(cfg), speculative_algorithm=cfg.speculative_algorithm, + pp_size=cfg.pp_size, ) @@ -36,6 +37,7 @@ def validate_layernorm_sp( tp_size: int, attn_dp_enabled: bool, speculative_algorithm: Optional[str], + pp_size: int = 1, ) -> None: """Fail loud for unsupported / incompatible configs. Callers gate on the flag.""" from sglang.srt.layers.layernorm_sp import SP_SUPPORTED_ARCHITECTURES @@ -62,3 +64,9 @@ def validate_layernorm_sp( "(EAGLE/EAGLE3): the captured aux hidden states would be " "sequence-sharded." ) + if pp_size > 1: + raise ValueError( + "--enable-layernorm-sp is not compatible with pipeline parallelism: " + "only the first rank's stack starts the sequence-sharded region, so " + "the next rank would receive a stream it does not expect." + ) diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 06899642ab..9ed2279041 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -36,7 +36,7 @@ from sglang.srt.layers.layer_boundary.contracts import ( StageKind, StagePath, ) -from sglang.srt.layers.layer_boundary.exit import ExitPolicy +from sglang.srt.layers.layer_boundary.exit import ExitPolicy, exit_facts from sglang.srt.layers.layer_boundary.fusions.allreduce import ( attn_input_fusions, ffn_input_fusions, @@ -191,7 +191,7 @@ class StagePlan: attn_tp_gather=exit_gather, ) ) - self.paths[variant] = StagePath( + path = StagePath( entry=entry, output=edges.outgoing.produced, output_move=out.output_move, @@ -200,6 +200,9 @@ class StagePlan: returns_over_dp=out.returns_over_dp, writes_at_handoff=writes_at_handoff, ) + self.paths[variant] = msgspec.structs.replace( + path, exit=exit_facts(kind, self, variant, path) + ) @property def incoming_residual_rows(self): diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 005cafb28a..9669fb4ff0 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -262,6 +262,48 @@ class EntryPath(msgspec.Struct, frozen=True): capture_preserves_residual: Optional[Callable] = None +class ExitFacts(msgspec.Struct, frozen=True): + """What an FFN's or a mixer's exit decides from fixed facts, for one batch + variant: the parallel configuration, the declarations and the bound path. + What depends on the batch (its padding, whether the attention-DP + reduce-scatter is usable, whether it has tokens, a fused consumer's + accept) stays with the exit's decision for that batch. + + They are read once, when the stage binds, under the scope it is built + in: a speculative draft's own MoE backends and boundary reduction while + it builds. A worker builds and runs its draft under the same MoE backend + scopes, so they hold for every forward. Whether the attention-DP + reduce-scatter is usable can change with elastic EP, so it is not one of + them. + + Fields: + may_defer_sum: Whether an FFN may leave its sum to the next input as + far as fixed facts go: TP > 1, not the stack's end, one group the + next input can complete it over, no MoE-CP all-gather or + input-scattered batch, no EAGLE draft under attention DP, and under + attention DP only when the next input also scatters the output back. + defers_mixer_sum: Whether a mixer carries its sum to the next attention. + sum_in_reduce_scatter: Whether, without an attention-DP reduce-scatter, + a reduce-scatter completes the FFN's sum: its exit's own move, or + the next input's on an input-scattered batch. + sum_left_to_next_input: The sum an input-scattered batch's next input + completes, as the stream records it. + reduce_scatterv: Whether the attention-DP return is the reduce-scatterv. + single_sum: Whether the post-expert sum is one all-reduce, which a + deferred sum's consumer completes. + fused_consumer_sum: Whether a LoRA or TP1 shared-expert output, which + is not one all-reduce, may still be left to a fused consumer. + """ + + may_defer_sum: bool = False + defers_mixer_sum: bool = False + sum_in_reduce_scatter: bool = False + sum_left_to_next_input: Optional[SumGroup] = None + reduce_scatterv: bool = False + single_sum: bool = False + fused_consumer_sum: bool = False + + class StagePath(msgspec.Struct, frozen=True): """Precomputed entry and exit work for one stage and batch variant. @@ -276,6 +318,7 @@ class StagePath(msgspec.Struct, frozen=True): returns_over_dp: Whether output uses batch-dependent attention-DP transport. writes_at_handoff: Whether the exit completes the output and writes it into the residual, for an FFN that hands off to another pipeline rank. + exit: What the exit decides from fixed facts (see ExitFacts). """ entry: EntryPath @@ -287,6 +330,7 @@ class StagePath(msgspec.Struct, frozen=True): returns_over_dp: bool = False writes_at_handoff: bool = False + exit: ExitFacts = ExitFacts() class StageKind(Enum): diff --git a/python/sglang/srt/layers/layer_boundary/exit.py b/python/sglang/srt/layers/layer_boundary/exit.py index a47f2cb409..0e933f6242 100644 --- a/python/sglang/srt/layers/layer_boundary/exit.py +++ b/python/sglang/srt/layers/layer_boundary/exit.py @@ -28,7 +28,11 @@ from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, is_enable_moe_cp_allgather, ) -from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context +from sglang.srt.layers.layer_boundary.contracts import ( + BatchVariant, + ExitFacts, + StageKind, +) from sglang.srt.layers.layer_boundary.layout import ( SumGroup, _ffn_has_tokens, @@ -72,13 +76,14 @@ def _select_dp_reduce_scatter( forward_batch: ForwardBatch, *, may_reduce_scatter: bool, - may_reduce_scatterv: bool, + reduce_scatterv: bool, ) -> Optional[Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]]: """The reduce-scatter that brings an FFN output gathered over attention DP back to this rank's tokens when the FFN leaves its sum to it (see OutputContract); None when the FFN reduces the output and only a scatter - remains.""" - if should_use_dp_reduce_scatterv() and may_reduce_scatterv: + remains. Whether the reduce-scatter is usable follows the DP group, which + an elastic scale-up changes, so it is read for each batch.""" + if reduce_scatterv: return dp_reduce_scatterv if ( may_reduce_scatter @@ -89,6 +94,98 @@ def _select_dp_reduce_scatter( return None +def exit_facts(kind, plan, variant, path) -> ExitFacts: + """What ``path``'s exit decides from fixed facts on ``variant``: the + parallel configuration, the declarations and the bound path. A stage + without an exit (an attention that always leaves its sum) decides + nothing. A condition is read only where the exit used to read it.""" + if plan.finishes_directly: + return ExitFacts() + output = path.output + scattered = variant is BatchVariant.INPUT_SCATTERED + tp = get_parallel().tp_size > 1 + if kind is StageKind.ATTENTION: + return ExitFacts( + defers_mixer_sum=output.may_defer_to_next + and tp + and not is_dp_attention_enabled() + and _sum_deferral_allowed(plan, output, scattered=scattered) + ) + sum_in_reduce_scatter = output.may_reduce_scatter and ( + path.output_move_completes_sum or (scattered and not plan.terminal) + ) + may_defer_sum = ( + tp + and not plan.terminal + and _sum_deferral_allowed(plan, output, scattered=scattered) + # Under attention DP the next layer must also run postprocess's + # scatter back to this rank's tokens, and nothing more. + and ( + not is_dp_attention_enabled() + or (path.returns_over_dp and not output.update.applied_at_exit) + ) + ) + single_sum = may_defer_sum and post_experts_sum_is_one_all_reduce() + return ExitFacts( + may_defer_sum=may_defer_sum, + sum_in_reduce_scatter=sum_in_reduce_scatter, + # On an input-scattered batch the next input's TP reduce-scatter + # completes the sum; the other reduce-scatters are this exit's move. + sum_left_to_next_input=( + SumGroup.TP + if sum_in_reduce_scatter + and not path.returns_over_dp + and not path.output_move_completes_sum + else None + ), + reduce_scatterv=path.returns_over_dp + and should_use_dp_reduce_scatterv() + and output.may_reduce_scatterv, + single_sum=single_sum, + fused_consumer_sum=may_defer_sum and not single_sum and _fused_consumer_sum(), + ) + + +def _sum_deferral_allowed(plan, output, *, scattered: bool) -> bool: + # Under the MoE-CP all-gather the fusion path would skip complete_now + # and its MoE-CP scatter, leaving hidden_states longer than the residual. + if is_enable_moe_cp_allgather() or output.group is None: + return False + + # The fused residual+LN reduces over a single group. Hybrid EP+TP spans + # two disjoint groups; post_experts_all_reduce() merges them into one + # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb. + # When merging is blocked, no single group covers both, so fusion stays off. + parallel = get_parallel() + if ( + parallel.moe_ep_size > 1 + and parallel.moe_tp_size > 1 + and not can_merge_post_experts_all_reduce() + ): + return False + + if ( + is_dp_attention_enabled() + and plan._speculative_algo is not None + and plan._speculative_algo.is_eagle() + ): + return False + + return not scattered + + +def _fused_consumer_sum() -> bool: + # LoRA-B is replicated and linear. TP1 shared experts add on rank zero + # when the sum is deferred. Preserve these fused paths, but keep their + # producer-side reduction order when no fused consumer is enabled. + return ( + (get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get()) + and get_moe_a2a_backend().is_none() + and not get_exec().comm.enable_quant_communications + and post_experts_reduction_group() is get_parallel().tp_group + ) + + class ExitPolicy: """Choose and finish one producer's output; the batch owns its residual.""" @@ -133,18 +230,6 @@ class ExitPolicy: return stream.write(hidden_states) return stream.record(hidden_states, update, declared_sum=declared_sum) - @staticmethod - def _sum_left_to_next_input(steps, sum_in_reduce_scatter): - # On an input-scattered batch the next input's TP reduce-scatter - # completes the sum; the other reduce-scatters are this exit's move. - if ( - sum_in_reduce_scatter - and not steps.returns_over_dp - and not steps.output_move_completes_sum - ): - return SumGroup.TP - return None - def _next_input_can_scatter(self, steps) -> bool: """Whether the next layer's input can run this layer's move of its FFN output back to this rank's tokens: the base postprocess scatter, when @@ -162,19 +247,13 @@ class ExitPolicy: return _select_dp_reduce_scatter( forward_batch, may_reduce_scatter=steps.output.may_reduce_scatter, - may_reduce_scatterv=steps.output.may_reduce_scatterv, + reduce_scatterv=steps.exit.reduce_scatterv, ) def _sum_in_reduce_scatter(self, steps, dp_step: Optional[Callable]) -> bool: """Whether a reduce-scatter completes the FFN output's sum: the attention-DP one ``dp_step`` names, or the CP / input-scattered one.""" - if dp_step is not None: - return True - if not steps.output.may_reduce_scatter: - return False - if steps.output_move_completes_sum: - return True - return get_attn_tp_context().input_scattered and not self.plan.terminal + return dp_step is not None or steps.exit.sum_in_reduce_scatter def ffn_reduction_group(self, steps) -> GroupCoordinator: """The group this layer's FFN output owes its sum over: the MoE output's @@ -307,33 +386,6 @@ class ExitPolicy: off its MoE finalize, then call ``finish``.""" return FfnExit(self, forward_batch, stream=stream) - def _sum_deferral_allowed(self, steps) -> bool: - # Under the MoE-CP all-gather the fusion path would skip complete_now - # and its MoE-CP scatter, leaving hidden_states longer than the residual. - if is_enable_moe_cp_allgather() or steps.output.group is None: - return False - - # The fused residual+LN reduces over a single group. Hybrid EP+TP spans - # two disjoint groups; post_experts_all_reduce() merges them into one - # _TP reduction when moe_dp_size == 1, which the fused kernel can absorb. - # When merging is blocked, no single group covers both, so fusion stays off. - parallel = get_parallel() - if ( - parallel.moe_ep_size > 1 - and parallel.moe_tp_size > 1 - and not can_merge_post_experts_all_reduce() - ): - return False - - if ( - is_dp_attention_enabled() - and self.plan._speculative_algo is not None - and self.plan._speculative_algo.is_eagle() - ): - return False - - return not get_attn_tp_context().input_scattered - def _defers_sum( self, forward_batch: ForwardBatch, @@ -344,23 +396,20 @@ class ExitPolicy: ) -> bool: """Whether the FFN leaves its output's all-reduce to the next layer's input when no fused kernel takes it: when the next layer would run the - same all-reduce the FFN itself would have.""" + same all-reduce the FFN itself would have. A reduce-scatter completing + the sum rules it out, and under attention DP so does any move back but + the scatter the next input runs.""" return ( - get_parallel().tp_size > 1 - and not self.plan.terminal - and self._sum_deferral_allowed(steps) - and _batch_allows_deferred_sum(forward_batch, self.plan) + steps.exit.may_defer_sum + and _batch_allows_deferred_sum(forward_batch, steps.exit, self.plan) and not sum_in_reduce_scatter - # Under attention DP the next layer must also run postprocess's - # scatter back to this rank's tokens, and nothing more. - and ( - not is_dp_attention_enabled() - or (self._next_input_can_scatter(steps) and dp_step is None) - ) + and dp_step is None ) -def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bool: +def _batch_allows_deferred_sum( + forward_batch: ForwardBatch, facts: ExitFacts, boundary=None +) -> bool: """Admit ordinary single-sum outputs, plus LoRA and TP1 shared-expert outputs when a fused consumer (the backend's can_defer_all_reduce, FlashInfer or aiter) can take them. @@ -370,17 +419,9 @@ def _batch_allows_deferred_sum(forward_batch: ForwardBatch, boundary=None) -> bo """ if not _ffn_has_tokens(forward_batch): return False - if post_experts_sum_is_one_all_reduce(): + if facts.single_sum: return True - # LoRA-B is replicated and linear. TP1 shared experts add on rank zero - # when the sum is deferred. Preserve these fused paths, but keep their - # producer-side reduction order when no fused consumer is enabled. - if not ( - (get_lora().enable_lora or envs.SGLANG_SHARED_EXPERT_TP1.get()) - and get_moe_a2a_backend().is_none() - and not get_exec().comm.enable_quant_communications - and post_experts_reduction_group() is get_parallel().tp_group - ): + if not facts.fused_consumer_sum: return False if ( boundary is not None @@ -452,12 +493,7 @@ class MixerExit: self._update = produced.update self._group = produced.group self._declared_sum = produced.group if produced.always_partial else None - self._defers = ( - produced.may_defer_to_next - and get_parallel().tp_size > 1 - and not is_dp_attention_enabled() - and boundary._sum_deferral_allowed(steps) - ) + self._defers = steps.exit.defers_mixer_sum def __enter__(self) -> MixerExit: return self @@ -519,14 +555,16 @@ class FfnExit: ): self._stream = stream self.boundary = boundary + # The same path the entry took: the flags that select it hold for the + # whole call, and a stage keeps no state between its entry and exit. steps = boundary.plan.path_for(forward_batch) completion = boundary._decide(forward_batch, steps) self.defer_moe_finalize = completion.defer_moe_finalize self._complete = completion.complete self._update = steps.output.update - self._declared_sum = boundary._sum_left_to_next_input( - steps, completion.sum_in_reduce_scatter - ) + # A reduce-scatter that completes the sum on the way back is the + # next input's only on an input-scattered batch. + self._declared_sum = steps.exit.sum_left_to_next_input self._scope = get_forward().scoped(defer_moe_finalize=self.defer_moe_finalize) def __enter__(self) -> FfnExit: diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 42f7f3a659..424cf1758f 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -317,6 +317,13 @@ class StageBoundary: return hidden_states def _prepare_input(self, hidden_states, stream, forward_batch, **call): + hidden_states, steps = self._select(hidden_states, forward_batch) + return self._prepare(hidden_states, stream, forward_batch, steps, **call) + + def _select(self, hidden_states, forward_batch): + """This call's path, looked up once the flags that select it are set: + the stack's first stage starts a sequence-parallel batch. Publishes + the rows LoRA reads for the stage's input.""" plan = self.plan if self.kind is StageKind.ATTENTION: publish_attn_lora_rows(plan._publish_lora_layout) @@ -329,11 +336,10 @@ class StageBoundary: ) if get_forward().sp_active: hidden_states = layernorm_sp.sp_entry_scatter(hidden_states) - else: - publish_ffn_lora_rows( - plan._publish_lora_layout, self.entry(forward_batch).input_rows - ) - return self._prepare(hidden_states, stream, forward_batch, **call) + steps = plan.path_for(forward_batch) + if self.kind is StageKind.FFN: + publish_ffn_lora_rows(plan._publish_lora_layout, steps.entry.input_rows) + return hidden_states, steps def _prepare_attention( self, @@ -362,14 +368,16 @@ class StageBoundary: capture(value) else: capture(value + previous, owned=True) - hidden_states, stream = self._prepare_input( + hidden_states, steps = self._select(hidden_states, forward_batch) + hidden_states, stream = self._prepare( hidden_states, stream, forward_batch, + steps, quant_format=quant_format, post_residual_addition=post_residual_addition, ) - entry = self.entry(forward_batch) + entry = steps.entry if capture is not None and not capture_before_read: value = stream.residual move = entry.capture_move diff --git a/python/sglang/test/boundary_fixtures.py b/python/sglang/test/boundary_fixtures.py index 33d7ccd43e..e810ea5655 100644 --- a/python/sglang/test/boundary_fixtures.py +++ b/python/sglang/test/boundary_fixtures.py @@ -98,6 +98,7 @@ def stub_plan(): plan._unpadded_attn_tp_size = None plan.paths = {} plan.enters_stack = False + plan.finishes_directly = False return plan diff --git a/test/registered/unit/layer_boundary/test_aux_hidden_states.py b/test/registered/unit/layer_boundary/test_aux_hidden_states.py index 5a2b33f933..9b4688038a 100644 --- a/test/registered/unit/layer_boundary/test_aux_hidden_states.py +++ b/test/registered/unit/layer_boundary/test_aux_hidden_states.py @@ -92,17 +92,17 @@ class TestAuxStorage(CustomTestCase): def boundary(self, stream, *, move=None): boundary = stub_plan() boundary.norm = None - stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock( - return_value=(stream.residual, stream) - ) - stub_stage(boundary, StageKind.ATTENTION).entry = lambda _: SimpleNamespace( + stage = stub_stage(boundary, StageKind.ATTENTION) + stage._prepare = Mock(return_value=(stream.residual, stream)) + entry = SimpleNamespace( input_move=move, input_retainable=False, capture_move=None, capture_move_allocates=False, capture_preserves_residual=None, ) - return stub_stage(boundary, StageKind.ATTENTION) + stage._select = lambda hidden, batch: (hidden, SimpleNamespace(entry=entry)) + return stage def test_packer_receives_borrowed_boundary_value_without_intermediate_clone(self): for callback in (False, True): @@ -162,12 +162,12 @@ class TestAuxStorage(CustomTestCase): outputs = AuxHiddenStatePacker(1) stage = self.boundary(stream) - def prepare(value, stream, batch, **kwargs): + def prepare(value, stream, batch, steps, **kwargs): value.zero_() stream.write(value) return value, stream - stage._prepare_input = prepare + stage._prepare = prepare with patch.object( torch.Tensor, "clone", side_effect=AssertionError("extra clone") ): diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index 60a19b5589..c7208c2aed 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -14,7 +14,11 @@ from sglang.srt.layers.layer_boundary import ( ) from sglang.srt.layers.layer_boundary import exit as exits from sglang.srt.layers.layer_boundary import stage as stages -from sglang.srt.layers.layer_boundary.contracts import BatchVariant +from sglang.srt.layers.layer_boundary.contracts import ( + BatchVariant, + ExitFacts, + StageKind, +) from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion from sglang.srt.layers.layer_boundary.layout import SumGroup from sglang.srt.layers.layer_boundary.residual import batch @@ -35,15 +39,21 @@ class TestBoundaryIntegrations(unittest.TestCase): produced = fixture.comm.OutputContract( rows, group=SumGroup.ATTN_TP, may_defer_to_next=True, update=PLAIN_ADD ) - plan = SimpleNamespace(path_for=lambda _: SimpleNamespace(output=produced)) + path = SimpleNamespace(output=produced) + plan = SimpleNamespace(finishes_directly=False, path_for=lambda _: path) boundary = exits.ExitPolicy(plan) - boundary._sum_deferral_allowed = lambda _: True summed = Mock(side_effect=lambda value, *args, **kwargs: value * 2) with ( fixture.planning(parallel), patch.object(exits, "is_dp_attention_enabled", return_value=True), patch.object(exits, "sum_output", summed), ): + # Under attention DP a mixer completes its sum even where it could + # otherwise carry it to the next attention. + with patch.object(exits, "_sum_deferral_allowed", return_value=True): + path.exit = exits.exit_facts( + StageKind.ATTENTION, plan, BatchVariant.ORDINARY, path + ) stream = ResidualStream(torch.zeros(3, 4)) result = exits.MixerExit(boundary, None, stream=stream) value = torch.ones(3, 4) @@ -156,9 +166,6 @@ class TestBoundaryIntegrations(unittest.TestCase): exits.envs.SGLANG_SHARED_EXPERT_TP1, "get", return_value=shared ), patch.object(exits, "_ffn_has_tokens", return_value=True), - patch.object( - exits, "post_experts_sum_is_one_all_reduce", return_value=False - ), patch.object( exits, "get_lora", @@ -189,8 +196,11 @@ class TestBoundaryIntegrations(unittest.TestCase): ), patch.object(exits, "aiter_ar_fusion_applies", return_value=False), ): + facts = ExitFacts( + single_sum=False, fused_consumer_sum=exits._fused_consumer_sum() + ) self.assertEqual( - exits._batch_allows_deferred_sum(fb), + exits._batch_allows_deferred_sum(fb, facts), enabled and (lora or shared), ) diff --git a/test/registered/unit/layer_boundary/test_boundary_output_policy.py b/test/registered/unit/layer_boundary/test_boundary_output_policy.py index defe9735dd..d87c169823 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_policy.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_policy.py @@ -125,9 +125,13 @@ class TestBoundaryOutputPolicy(unittest.TestCase): policy=policy, varlen=varlen, max_len=max_len, can_rs=can_rs ): parallel = fixture.parallel_of(attn_dp=2, attn_tp=1) - layer = fixture.build( - fixture.layer_case(2, 3), parallel, boundary_reduction=policy - ) + # Whether the DP return is the reduce-scatterv is the configuration's. + with patch_communicator( + "should_use_dp_reduce_scatterv", lambda: varlen + ): + layer = fixture.build( + fixture.layer_case(2, 3), parallel, boundary_reduction=policy + ) calls = [] def move(step, batch, value): @@ -147,7 +151,6 @@ class TestBoundaryOutputPolicy(unittest.TestCase): ) with ( fixture.planning(parallel), - patch_communicator("should_use_dp_reduce_scatterv", lambda: varlen), patch_communicator("can_use_dp_reduce_scatter", lambda: can_rs), patch_communicator("to_dp_local", move), patch_communicator("sum_output", summed), diff --git a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py index 1d447722ba..ac5597b92c 100644 --- a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py @@ -574,9 +574,6 @@ class TestDeferredLoraAllReduce(unittest.TestCase): for eligible, a2a in ((True, False), (False, False), (True, True)): with ( patch.object(exits, "_ffn_has_tokens", return_value=True), - patch.object( - exits, "post_experts_sum_is_one_all_reduce", return_value=False - ), patch.object( exits, "get_lora", return_value=SimpleNamespace(enable_lora=True) ), @@ -602,8 +599,11 @@ class TestDeferredLoraAllReduce(unittest.TestCase): ) as gate, ): # Avoid platform-specific Aiter details: no residual cannot fuse. + facts = exits.ExitFacts( + single_sum=False, fused_consumer_sum=exits._fused_consumer_sum() + ) with patch.object(exits, "_use_aiter", False, create=True): - result = exits._batch_allows_deferred_sum(fb, boundary) + result = exits._batch_allows_deferred_sum(fb, facts, boundary) self.assertEqual(result, eligible and not a2a) if a2a: gate.assert_not_called() diff --git a/test/registered/unit/layer_boundary/test_declared_attention_cp.py b/test/registered/unit/layer_boundary/test_declared_attention_cp.py index 499cd80f2a..a4d1a98243 100644 --- a/test/registered/unit/layer_boundary/test_declared_attention_cp.py +++ b/test/registered/unit/layer_boundary/test_declared_attention_cp.py @@ -125,6 +125,8 @@ class TestAttentionCpBoundary(CustomTestCase): ((comm, "get_moe_cp_size"), lambda: CP_SIZE), ((comm, "get_moe_cp_rank"), lambda: cp), ((comm, "should_use_dp_reduce_scatterv"), lambda: False), + ((comm, "post_experts_sum_is_one_all_reduce"), lambda: False), + ((comm, "get_lora"), lambda: SimpleNamespace(enable_lora=False)), ( (comm, "get_moe_a2a_backend"), lambda: SimpleNamespace(is_none=lambda: True), diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index c0b7d84677..376c8bcfa3 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -27,7 +27,6 @@ from sglang.srt.layers.layer_boundary import ( append_stages, ) from sglang.srt.layers.layer_boundary import boundary as comm_boundary -from sglang.srt.layers.layer_boundary import construction as comm_layer from sglang.srt.layers.layer_boundary import ( declare_attn, declare_ffn, @@ -150,10 +149,44 @@ def planning( patch_communicator( "get_exec", lambda: SimpleNamespace( - comm=SimpleNamespace(boundary_reduction=boundary_reduction), + comm=SimpleNamespace( + boundary_reduction=boundary_reduction, + enable_quant_communications=False, + ), overlap=SimpleNamespace(enable_two_batch_overlap=False), ), ), + # What the MoE predicates an exit's decisions read, in their module. + patch.object(moe_utils, "get_parallel", get_parallel), + patch.object( + moe_utils, + "get_moe_a2a_backend", + lambda: SimpleNamespace( + is_none=lambda: not a2a, + is_flashinfer=lambda: False, + is_pplx=lambda: False, + is_flashinfer_megamoe=lambda: False, + ), + ), + patch.object(moe_utils, "get_lora", lambda: SimpleNamespace(enable_lora=False)), + patch.object( + moe_utils, + "get_exec", + lambda: SimpleNamespace( + comm=SimpleNamespace(enable_quant_communications=False) + ), + ), + patch.object( + moe_utils, "post_experts_reduction_group", lambda: get_parallel().tp_group + ), + patch.object( + moe_utils, "should_use_flashinfer_cutlass_moe_fp4_allgather", lambda: False + ), + patch.object( + moe_utils, + "is_dp_attention_enabled", + lambda: get_parallel().attn_dp_size > 1, + ), ): yield @@ -401,11 +434,15 @@ class TestMhcOnTheDeclarations(CustomTestCase): ): with self.subTest(name): hc_post = MagicMock(side_effect=lambda h, r, h_res, h_post: h + r) - communicator = build_mhc( - layer_case(1, 3), - parallel, - hc_post=hc_post, - ) + # Whether the DP return is the reduce-scatterv is the configuration's. + with patch_communicator( + "should_use_dp_reduce_scatterv", lambda: reduce_scatterv + ): + communicator = build_mhc( + layer_case(1, 3), + parallel, + hc_post=hc_post, + ) communicator.mhc.h_res = communicator.mhc.h_post = torch.zeros(2) forward_batch = SimpleNamespace( dp_padding_mode=SimpleNamespace(is_max_len=lambda: is_max_len), @@ -415,9 +452,6 @@ class TestMhcOnTheDeclarations(CustomTestCase): to_local_tokens = MagicMock(side_effect=lambda step, fb, h: h[:2]) with ( planning(parallel), - patch_communicator( - "should_use_dp_reduce_scatterv", lambda: reduce_scatterv - ), patch_communicator("can_use_dp_reduce_scatter", lambda: True), patch_communicator("_select_dp_reduce_scatter", choose), patch_communicator("to_dp_local", to_local_tokens), @@ -877,14 +911,10 @@ class TestTheSequenceParallelRegion(CustomTestCase): ), patch_communicator("is_dp_attention_enabled", lambda: False), ): - self.assertIs( - communicator.ffn.plan.output._sum_deferral_allowed( - communicator.ffn.plan.output.plan.path_for( - SimpleNamespace(forward_mode=ForwardMode.DECODE) - ) - ), - not active, + steps = communicator.ffn.plan.path_for( + SimpleNamespace(forward_mode=ForwardMode.DECODE) ) + self.assertIs(steps.exit.may_defer_sum, not active) class TestInputScatteredAttention(CustomTestCase): diff --git a/test/registered/unit/layer_boundary/test_layernorm_sp.py b/test/registered/unit/layer_boundary/test_layernorm_sp.py index 6a70c0e62e..1b21feebcb 100644 --- a/test/registered/unit/layer_boundary/test_layernorm_sp.py +++ b/test/registered/unit/layer_boundary/test_layernorm_sp.py @@ -122,6 +122,10 @@ class TestLayerNormSPValidation(CustomTestCase): with self.assertRaisesRegex(ValueError, "speculative"): validate_layernorm_sp(**{**self.VALID, "speculative_algorithm": "EAGLE3"}) + def test_rejects_pipeline_parallelism(self): + with self.assertRaisesRegex(ValueError, "pipeline"): + validate_layernorm_sp(**{**self.VALID, "pp_size": 2}) + class _Norm: def __call__(self, x, residual=None, post_residual_addition=None): diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py index a6c1c8b025..22c5a16f39 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py @@ -16,6 +16,7 @@ from sglang.srt.layers.layer_boundary import ( UnreducedOutput, ) from sglang.srt.layers.layer_boundary import exit as exit_module +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, StageKind from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.models import nemotron_h_utils as utils @@ -190,18 +191,21 @@ class TestMixerExit(CustomTestCase): may_defer_to_next=may, update=PLAIN_ADD, ) - communicator = SimpleNamespace( - plan=SimpleNamespace( - path_for=lambda batch: SimpleNamespace(output=produced), - ), - _sum_deferral_allowed=MagicMock(return_value=movable), - ) + path = SimpleNamespace(output=produced) + plan = SimpleNamespace(finishes_directly=False, path_for=lambda b: path) + communicator = SimpleNamespace(plan=plan) hidden = torch.ones(2, 4) summed = MagicMock(side_effect=lambda h, *args, **kwargs: h * 2) with ( get_parallel().override(tp_group=tp_group, tp_size=2), patch.object(exit_module, "sum_output", summed), ): + with patch.object( + exit_module, "_sum_deferral_allowed", return_value=movable + ): + path.exit = exit_module.exit_facts( + StageKind.ATTENTION, plan, BatchVariant.ORDINARY, path + ) with MixerExit( communicator, None, stream=ResidualStream() ) as mixer_exit: diff --git a/test/registered/unit/layer_boundary/test_reduction_fusion.py b/test/registered/unit/layer_boundary/test_reduction_fusion.py index dd0b5a3a8f..aa213262ad 100644 --- a/test/registered/unit/layer_boundary/test_reduction_fusion.py +++ b/test/registered/unit/layer_boundary/test_reduction_fusion.py @@ -4,6 +4,7 @@ import unittest from functools import partial from unittest.mock import MagicMock, patch +import msgspec import torch from sglang.srt.layers import layer_boundary as comm @@ -14,8 +15,11 @@ from sglang.srt.layers.layer_boundary import ( ) from sglang.srt.layers.layer_boundary import ops as transport_ops from sglang.srt.layers.layer_boundary import prepare as comm_ops -from sglang.srt.layers.layer_boundary.contracts import BatchVariant -from sglang.srt.layers.layer_boundary.exit import ExitPolicy +from sglang.srt.layers.layer_boundary.contracts import BatchVariant, StageKind +from sglang.srt.layers.layer_boundary.exit import ( + _sum_deferral_allowed, + exit_facts, +) from sglang.srt.layers.layer_boundary.ops import keep_output from sglang.srt.layers.moe import ( can_merge_post_experts_all_reduce, @@ -359,12 +363,11 @@ class TestFuseMlpAllReduceGate(CustomTestCase): tp_size=moe_ep_size * moe_tp_size * moe_dp_size, ), ): - return ExitPolicy( - _fake_communicator(ffn_sum_is_movable) - )._sum_deferral_allowed( - ExitPolicy(_fake_communicator(ffn_sum_is_movable)).plan.path_for( - forward_batch - ) + communicator = _fake_communicator(ffn_sum_is_movable) + return _sum_deferral_allowed( + communicator, + communicator.path_for(forward_batch).output, + scattered=False, ) def test_hybrid_ep_tp_fuses_when_mergeable(self): @@ -518,6 +521,14 @@ class TestDeferFfnReduction(CustomTestCase): return_value=tp_group_object if tp_group else object(), ), ): + # What the exit decides from fixed facts is recorded on the path. + path = communicator.paths[BatchVariant.ORDINARY] + communicator.paths[BatchVariant.ORDINARY] = msgspec.structs.replace( + path, + exit=exit_facts( + StageKind.FFN, communicator, BatchVariant.ORDINARY, path + ), + ) return communicator.output._defers_sum( forward_batch, communicator.output.plan.path_for(forward_batch), diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index 62dd33b794..95570a9493 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -250,7 +250,7 @@ class TestResidualStream(CustomTestCase): events = [] outputs = AuxHiddenStateList() - def prepare(hidden, stream, batch, **kwargs): + def prepare(hidden, stream, batch, steps, **kwargs): events.append("add_norm") self.assertIsNone(stream.pending.owed) # A fused read returns both the norm output and its updated residual. @@ -263,14 +263,16 @@ class TestResidualStream(CustomTestCase): outputs.capture(value, owned=owned) boundary.norm = None - stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock( - side_effect=prepare - ) - stub_stage(boundary, StageKind.ATTENTION).entry = lambda batch: SimpleNamespace( + stub_stage(boundary, StageKind.ATTENTION)._prepare = Mock(side_effect=prepare) + entry = SimpleNamespace( capture_move=None, capture_move_allocates=False, capture_preserves_residual=None, ) + stub_stage(boundary, StageKind.ATTENTION)._select = lambda hidden, batch: ( + hidden, + SimpleNamespace(entry=entry), + ) output, stream = prepare_attention( stub_stage(boundary, StageKind.ATTENTION), self.hidden, @@ -280,7 +282,7 @@ class TestResidualStream(CustomTestCase): ) self.assertEqual(events, ["add_norm", "capture"]) self.group.all_reduce.assert_called_once() - stub_stage(boundary, StageKind.ATTENTION)._prepare_input.assert_called_once() + stub_stage(boundary, StageKind.ATTENTION)._prepare.assert_called_once() torch.testing.assert_close(output, torch.full((2, 4), 15.0)) torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) stream.residual.zero_() @@ -291,7 +293,7 @@ class TestResidualStream(CustomTestCase): extra = torch.full_like(self.partial, 7.0) outputs = AuxHiddenStateList() - def prepare(hidden, stream, batch, **kwargs): + def prepare(hidden, stream, batch, steps, **kwargs): self.assertEqual(len(outputs), 1) self.assertIs(kwargs["post_residual_addition"], extra) updated = hidden + stream.residual + extra @@ -299,14 +301,16 @@ class TestResidualStream(CustomTestCase): return updated * 3, stream boundary.norm = None - stub_stage(boundary, StageKind.ATTENTION)._prepare_input = Mock( - side_effect=prepare - ) - stub_stage(boundary, StageKind.ATTENTION).entry = lambda batch: SimpleNamespace( + stub_stage(boundary, StageKind.ATTENTION)._prepare = Mock(side_effect=prepare) + entry = SimpleNamespace( capture_move=None, capture_move_allocates=False, capture_preserves_residual=None, ) + stub_stage(boundary, StageKind.ATTENTION)._select = lambda hidden, batch: ( + hidden, + SimpleNamespace(entry=entry), + ) output, _ = prepare_attention( stub_stage(boundary, StageKind.ATTENTION), self.hidden,