diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index 832904d3a4..ce2745bfb3 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -22,6 +22,7 @@ import msgspec import torch from sglang.srt.distributed import ( + GroupCoordinator, attention_tensor_model_parallel_all_reduce, attention_tensor_model_parallel_quant_all_reduce, tensor_model_parallel_all_reduce, @@ -626,17 +627,22 @@ def _attn_input_update_and_read_residual(quant_format: str): class UnreducedOutput(msgspec.Struct, frozen=True): """A layer output that still owes its FFN all-reduce, left for the next - layer's input norm. Hand it to the next layer, or pass it through - reduce_output() before reading it any other way.""" + layer's input. Hand it to the next layer, or pass it through reduce_output() + before reading it any other way.""" partial: torch.Tensor + # Under attention DP: the reduce-scatter that sums ``partial`` over TP while + # bringing it back to this rank's tokens, in place of the all-reduce. + reduce_and_redistribute: Optional[Callable[[torch.Tensor], torch.Tensor]] = None def reduce_output( hidden_states: Union[torch.Tensor, UnreducedOutput, None], ) -> Optional[torch.Tensor]: - """Run the all-reduce an UnreducedOutput still owes; pass anything else through.""" + """Run the reduction an UnreducedOutput still owes; pass anything else through.""" if isinstance(hidden_states, UnreducedOutput): + if hidden_states.reduce_and_redistribute is not None: + return hidden_states.reduce_and_redistribute(hidden_states.partial) return deferred_post_experts_all_reduce(hidden_states.partial) return hidden_states @@ -667,16 +673,21 @@ def _deferred_reduction_runs_on_the_tp_group() -> bool: return group is parallel.tp_group +def _ffn_has_tokens(forward_batch: ForwardBatch) -> bool: + if is_dp_attention_enabled(): + # The FFN runs on every DP rank's tokens, so every rank decides alike. + return (getattr(forward_batch, "global_dp_buffer_len", None) or 0) > 0 + return _batch_size(forward_batch) > 0 + + def _unfused_completion_matches_the_ffn(forward_batch: ForwardBatch) -> bool: """Whether deferred_post_experts_all_reduce is the all-reduce the FFN would have run itself: one full-precision sum over the same TP group, on an output - that is a plain partial sum, with no attention DP layout to restore. The - output is not a plain partial sum when the MoE combine already summed it, a - replicated shared expert is added after the reduction, or LoRA-B runs on - the unreduced activations.""" + that is a plain partial sum. The output is not a plain partial sum when the + MoE combine already summed it, a replicated shared expert is added after the + reduction, or LoRA-B runs on the unreduced activations.""" return ( - _batch_size(forward_batch) > 0 - and not is_dp_attention_enabled() + _ffn_has_tokens(forward_batch) and get_moe_a2a_backend().is_none() and not post_experts_output_is_complete(is_tp_path=True) and not get_exec().comm.enable_quant_communications @@ -833,6 +844,13 @@ class LayerCommunicator: quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, ): + if ( + isinstance(hidden_states, UnreducedOutput) + and hidden_states.reduce_and_redistribute is not None + ): + # No fused kernel runs under attention DP: the reduce-scatter back to + # this rank's tokens comes first. + hidden_states = reduce_output(hidden_states) pending = isinstance(hidden_states, UnreducedOutput) if pending: hidden_states = hidden_states.partial @@ -994,6 +1012,28 @@ class LayerCommunicator: is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, ) + def _reduce_scatter_for_next_layer( + self, forward_batch: ForwardBatch + ) -> Optional[Callable[[torch.Tensor], torch.Tensor]]: + """The reduce-scatter back to this rank's tokens that postprocess would + run on this layer's FFN output, bound for the next layer's input to run + instead; None when postprocess runs anything else.""" + if ( + self.is_last_layer + or (self._sp_variant is not None and get_forward().sp_active) + or self._communicate_summable_tensor_pair_fn + is not CommunicateSummableTensorPairFn._scatter_hidden_states + ): + return None + step = _output_to_local_tokens_step( + forward_batch, + allow_reduce_scatter=self.allow_reduce_scatter, + is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + ) + if step is _redistribute_output: + return None + return partial(_to_local_tokens, step, forward_batch) + def ffn_exit(self, forward_batch: ForwardBatch) -> "FfnExit": """Decide once how this layer's FFN output reduction completes. Use the result as a context manager around the FFN call, then call ``finish``.""" @@ -1111,8 +1151,41 @@ class LayerCommunicator: and self._ffn_sum_can_move_to_next_layer() and _unfused_completion_matches_the_ffn(forward_batch) and not self.should_use_reduce_scatter(forward_batch) + and self._next_layer_input_can_restore_the_layout(forward_batch) ) + def _next_layer_input_can_restore_the_layout( + self, forward_batch: ForwardBatch + ) -> bool: + """Whether the next layer's input can also run what postprocess would + after the FFN's own all-reduce: under attention DP, the scatter back to + this rank's tokens.""" + if not is_dp_attention_enabled(): + return True + return ( + self._communicate_summable_tensor_pair_fn + is CommunicateSummableTensorPairFn._scatter_hidden_states + and not (self._sp_variant is not None and get_forward().sp_active) + and _output_to_local_tokens_step( + forward_batch, + allow_reduce_scatter=self.allow_reduce_scatter, + is_layer_sparse=self.layer_scatter_modes.is_layer_sparse, + ) + is _redistribute_output + ) + + def _scatter_for_next_layer( + self, forward_batch: ForwardBatch + ) -> Optional[Callable[[torch.Tensor], torch.Tensor]]: + """Under attention DP, the all-reduce and the scatter back to this rank's + tokens that the next layer's input runs on a sum this layer's FFN left.""" + if ( + self._communicate_summable_tensor_pair_fn + is not CommunicateSummableTensorPairFn._scatter_hidden_states + ): + return None + return partial(_all_reduce_then_to_local_tokens, forward_batch) + # MOE_FULL gathers across the MoE-CP group, which spans every CP rank when CP is on. _SCATTER_MODE_SHARDED_AXES = { @@ -1181,13 +1254,31 @@ class FfnExit: def finish( self, hidden_states: torch.Tensor, residual: torch.Tensor ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: - """Leave the reduction to the next layer's input norm, or postprocess.""" + """Leave the reduction to the next layer's input, or postprocess.""" if not isinstance(hidden_states, torch.Tensor): # A deferred MoE finalize handoff, consumed by the next prepare_attn. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" return hidden_states, residual if self.fuse_mlp_allreduce: - return UnreducedOutput(hidden_states), residual + return ( + UnreducedOutput( + hidden_states, + reduce_and_redistribute=self.communicator._scatter_for_next_layer( + self.forward_batch + ), + ), + residual, + ) + reduce_and_redistribute = self.communicator._reduce_scatter_for_next_layer( + self.forward_batch + ) + if reduce_and_redistribute is not None: + return ( + UnreducedOutput( + hidden_states, reduce_and_redistribute=reduce_and_redistribute + ), + residual, + ) return self.communicator.postprocess_layer( hidden_states, residual, self.forward_batch ) @@ -1614,6 +1705,81 @@ class CommunicateWithAllReduceAndLayerNormFn: return hidden_states, residual +def _dp_scatter_group() -> GroupCoordinator: + parallel = get_parallel() + if parallel.tp_size == parallel.attn_dp_size: + return parallel.tp_group + return parallel.attn_tp_group + + +def _reduce_and_redistribute_output_varlen( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + get_parallel().tp_group.reduce_scatterv( + hidden_states, + output=local_hidden_states, + sizes=get_dp_global_num_tokens(), + ) + + +def _reduce_and_redistribute_output_max_len( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + dp_reduce_scatter_tensor(local_hidden_states, hidden_states) + + +def _redistribute_output( + local_hidden_states: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, +) -> None: + dp_scatter(local_hidden_states, hidden_states, forward_batch) + + +def _output_to_local_tokens_step( + forward_batch: ForwardBatch, *, allow_reduce_scatter: bool, is_layer_sparse: bool +) -> Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None]: + """How a FULL-layout layer output comes back to this rank's tokens under + attention DP: a reduce-scatter when the FFN left its sum to it, otherwise a + scatter of the reduced output.""" + # A MoE block leaves its sum to reduce_scatterv whenever it applies + # (should_skip_post_experts_all_reduce); a dense MLP does only under the + # published mlp_reduce_scatter, which needs allow_reduce_scatter. + if should_use_dp_reduce_scatterv() and (allow_reduce_scatter or is_layer_sparse): + return _reduce_and_redistribute_output_varlen + if ( + allow_reduce_scatter + and forward_batch.dp_padding_mode.is_max_len() + and can_use_dp_reduce_scatter() + ): + return _reduce_and_redistribute_output_max_len + return _redistribute_output + + +def _to_local_tokens( + step: Callable[[torch.Tensor, torch.Tensor, ForwardBatch], None], + forward_batch: ForwardBatch, + hidden_states: torch.Tensor, +) -> torch.Tensor: + local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) + step(local_hidden_states, hidden_states, forward_batch) + return local_hidden_states + + +def _all_reduce_then_to_local_tokens( + forward_batch: ForwardBatch, hidden_states: torch.Tensor +) -> torch.Tensor: + return _to_local_tokens( + _redistribute_output, + forward_batch, + deferred_post_experts_all_reduce(hidden_states), + ) + + class CommunicateSummableTensorPairFn: """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" @@ -1693,34 +1859,14 @@ class CommunicateSummableTensorPairFn: allow_reduce_scatter: bool = False, is_layer_sparse: bool = False, ): - if get_parallel().tp_size == get_parallel().attn_dp_size: - group = get_parallel().tp_group - else: - group = get_parallel().attn_tp_group - hidden_states, global_hidden_states = ( - get_local_dp_buffer(group), - hidden_states, + local_hidden_states = get_local_dp_buffer(_dp_scatter_group()) + step = _output_to_local_tokens_step( + forward_batch, + allow_reduce_scatter=allow_reduce_scatter, + is_layer_sparse=is_layer_sparse, ) - # A MoE block leaves its sum to reduce_scatterv whenever it applies - # (should_skip_post_experts_all_reduce); a dense MLP does only under the - # published mlp_reduce_scatter, which needs allow_reduce_scatter. - if should_use_dp_reduce_scatterv() and ( - allow_reduce_scatter or is_layer_sparse - ): - get_parallel().tp_group.reduce_scatterv( - global_hidden_states, - output=hidden_states, - sizes=get_dp_global_num_tokens(), - ) - elif ( - allow_reduce_scatter - and forward_batch.dp_padding_mode.is_max_len() - and can_use_dp_reduce_scatter() - ): - dp_reduce_scatter_tensor(hidden_states, global_hidden_states) - else: - dp_scatter(hidden_states, global_hidden_states, forward_batch) - return hidden_states, residual + step(local_hidden_states, hidden_states, forward_batch) + return local_hidden_states, residual @staticmethod def _gather( diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index eff958b42a..88dd8cd118 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -9,7 +9,6 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap import two_batch_overlap as tbo -from sglang.srt.layers import communicator from sglang.srt.layers.communicator import ScatterMode, UnreducedOutput from sglang.srt.utils import empty_context from sglang.test.ci.ci_register import register_cpu_ci @@ -20,8 +19,12 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") class TestTboEntryReducesItsInput(CustomTestCase): def test_split_and_merge_see_the_reduced_tensor(self): + # Under attention DP the partial sum spans every DP rank's tokens; its + # reduction also brings it back to this rank's three. reduced = torch.full((3, 4), 2.0) - hidden_states = UnreducedOutput(torch.ones(3, 4)) + hidden_states = UnreducedOutput( + torch.ones(6, 4), reduce_and_redistribute=lambda partial: reduced + ) seen = {} def split(**kwargs): @@ -33,9 +36,6 @@ class TestTboEntryReducesItsInput(CustomTestCase): return None, None with ( - patch.object( - communicator, "deferred_post_experts_all_reduce", lambda x: reduced - ), patch.object(tbo, "_model_forward_tbo_split_inputs", split), patch.object( tbo, "execute_overlapped_operations", lambda **kwargs: [{}, {}] diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 39bba99f2a..43353cb132 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -1,9 +1,15 @@ +import ast +import types import unittest +from pathlib import Path from unittest.mock import MagicMock, patch import torch +import sglang +from sglang.srt.layers import communicator as comm from sglang.srt.layers.communicator import ( + CommunicateSummableTensorPairFn, LayerCommunicator, UnreducedOutput, reduce_output, @@ -15,13 +21,24 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") -def make_communicator(*, fuse, reduce_scatter, cls=LayerCommunicator): +def make_communicator( + *, + fuse, + reduce_scatter, + next_layer_reduce_scatter=None, + next_layer_scatter=None, + cls=LayerCommunicator, +): """A communicator whose decisions and postprocess are stubbed, built without the process-wide parallel state.""" communicator = cls.__new__(cls) if cls is LayerCommunicator: communicator.should_defer_ffn_reduction = MagicMock(return_value=fuse) communicator.should_use_reduce_scatter = MagicMock(return_value=reduce_scatter) + communicator._reduce_scatter_for_next_layer = MagicMock( + return_value=next_layer_reduce_scatter + ) + communicator._scatter_for_next_layer = MagicMock(return_value=next_layer_scatter) communicator.postprocess_layer = MagicMock( side_effect=lambda hidden_states, residual, forward_batch: ( hidden_states + 1, @@ -56,6 +73,32 @@ class TestFfnExit(CustomTestCase): self.assertIs(residual, self.residual) communicator.postprocess_layer.assert_not_called() + def test_a_reduce_scatter_is_left_to_the_next_layer(self): + """Under attention DP the reduce-scatter postprocess would run goes to the + next layer with the partial sum.""" + step = MagicMock() + communicator = make_communicator( + fuse=False, reduce_scatter=True, next_layer_reduce_scatter=step + ) + seen, (hidden_states, residual) = self.run_exit(communicator) + self.assertEqual(seen, (False, True)) + self.assertIsInstance(hidden_states, UnreducedOutput) + self.assertIs(hidden_states.reduce_and_redistribute, step) + self.assertIs(residual, self.residual) + communicator.postprocess_layer.assert_not_called() + step.assert_not_called() + + def test_a_deferred_sum_carries_the_scatter_back_under_attention_dp(self): + step = MagicMock() + communicator = make_communicator( + fuse=True, reduce_scatter=False, next_layer_scatter=step + ) + _, (hidden_states, _) = self.run_exit(communicator) + self.assertIsInstance(hidden_states, UnreducedOutput) + self.assertIs(hidden_states.reduce_and_redistribute, step) + communicator._reduce_scatter_for_next_layer.assert_not_called() + communicator.postprocess_layer.assert_not_called() + def test_postprocess_completes_other_exits(self): for reduce_scatter in (False, True): with self.subTest(reduce_scatter=reduce_scatter): @@ -153,6 +196,17 @@ class TestReduceOutput(CustomTestCase): reduce_output(hidden_states) self.all_reduce.assert_called_once() + def test_a_reduce_scatter_left_by_the_last_layer_runs_once(self): + local = torch.full((1, 4), 7.0) + step = MagicMock(return_value=local) + partial = torch.ones(3, 4) + hidden_states = reduce_output( + UnreducedOutput(partial, reduce_and_redistribute=step) + ) + step.assert_called_once_with(partial) + self.assertIs(hidden_states, local) + self.all_reduce.assert_not_called() + def test_complete_hidden_states_pass_through(self): communicator = make_communicator(fuse=False, reduce_scatter=False) with communicator.ffn_exit(object()) as ffn_exit: @@ -182,5 +236,228 @@ class TestReduceOutput(CustomTestCase): self.assertIs(residual_out, residual) +class TestReduceScatterForNextLayer(CustomTestCase): + """Which reduce-scatter the next layer runs in place of postprocess.""" + + def communicator(self, *, is_last_layer=False, pair_fn=None, sp_variant=None): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator.is_last_layer = is_last_layer + communicator._sp_variant = sp_variant + communicator._communicate_summable_tensor_pair_fn = ( + pair_fn or CommunicateSummableTensorPairFn._scatter_hidden_states + ) + communicator.allow_reduce_scatter = True + communicator.layer_scatter_modes = types.SimpleNamespace(is_layer_sparse=True) + return communicator + + def bound(self, communicator, step): + with patch.object(comm, "_output_to_local_tokens_step", return_value=step): + return communicator._reduce_scatter_for_next_layer(object()) + + def test_a_reduce_scatter_is_bound_for_the_next_layer(self): + for step in ( + comm._reduce_and_redistribute_output_varlen, + comm._reduce_and_redistribute_output_max_len, + ): + with self.subTest(step=step.__name__): + bound = self.bound(self.communicator(), step) + self.assertIs(bound.func, comm._to_local_tokens) + self.assertIs(bound.args[0], step) + + def test_postprocess_keeps_everything_else(self): + reduce_scatter = comm._reduce_and_redistribute_output_varlen + for name, communicator, step in ( + ("scatter only", self.communicator(), comm._redistribute_output), + ("last layer", self.communicator(is_last_layer=True), reduce_scatter), + ( + "other layout change", + self.communicator( + pair_fn=CommunicateSummableTensorPairFn._scatter_hidden_states_moe + ), + reduce_scatter, + ), + ): + with self.subTest(name): + self.assertIsNone(self.bound(communicator, step)) + + def test_postprocess_keeps_an_active_layernorm_sp_region(self): + communicator = self.communicator(sp_variant=object()) + with get_forward().scoped(sp_active=True): + self.assertIsNone( + self.bound(communicator, comm._reduce_and_redistribute_output_varlen) + ) + + +class TestScatterForNextLayer(CustomTestCase): + def communicator(self, pair_fn): + communicator = LayerCommunicator.__new__(LayerCommunicator) + communicator._communicate_summable_tensor_pair_fn = pair_fn + return communicator + + def test_only_a_scatter_back_to_local_tokens_is_carried(self): + forward_batch = object() + bound = self.communicator( + CommunicateSummableTensorPairFn._scatter_hidden_states + )._scatter_for_next_layer(forward_batch) + self.assertIs(bound.func, comm._all_reduce_then_to_local_tokens) + self.assertIs(bound.args[0], forward_batch) + self.assertIsNone( + self.communicator( + CommunicateSummableTensorPairFn._trivial + )._scatter_for_next_layer(forward_batch) + ) + + def test_the_all_reduce_runs_before_the_scatter(self): + calls = [] + partial = torch.ones(3, 4) + local = torch.empty(1, 4) + with ( + patch.object( + comm, + "deferred_post_experts_all_reduce", + side_effect=lambda x: calls.append(("all_reduce", x)) or x * 2, + ), + patch.object(comm, "_dp_scatter_group", return_value="group"), + patch.object( + comm, + "get_local_dp_buffer", + side_effect=lambda group: calls.append(("buffer", group)) or local, + ), + patch.object( + comm, + "dp_scatter", + side_effect=lambda out, full, fb: calls.append(("scatter", out)), + ), + ): + result = comm._all_reduce_then_to_local_tokens(object(), partial) + self.assertEqual([c[0] for c in calls], ["all_reduce", "buffer", "scatter"]) + self.assertIs(calls[0][1], partial) + self.assertIs(result, local) + + +SRT_DIR = Path(sglang.__file__).resolve().parent / "srt" +# Attributes a decoder layer holds its FFN, or parts of it, in. +FFN_ATTRS = {"mlp", "moe", "shared_expert", "shared_experts", "share_expert"} + + +def can_be_true(value, params): + """Whether a use_dp_attention_reduce value can turn it on: anything but + False or a function forwarding its own parameter of that name.""" + if isinstance(value, ast.Constant) and value.value is False: + return False + return not (isinstance(value, ast.Name) and value.id in params) + + +class SourceIndex: + def __init__(self): + self.paths = { + "sglang.srt." + str(p.relative_to(SRT_DIR))[:-3].replace("/", "."): p + for p in SRT_DIR.rglob("*.py") + } + self._trees = {} + + def tree(self, module): + if module not in self._trees: + self._trees[module] = ast.parse(self.paths[module].read_text()) + return self._trees[module] + + def resolve(self, module, name): + """The (module, ClassDef) that `name` refers to in `module`, if any.""" + for node in self.tree(module).body: + if isinstance(node, ast.ClassDef) and node.name == name: + return module, node + for node in ast.walk(self.tree(module)): + if isinstance(node, ast.ImportFrom) and node.module in self.paths: + for alias in node.names: + if (alias.asname or alias.name) == name: + return self.resolve(node.module, alias.name) + return None + + +def submodule_constructions(cls): + """(attribute, class name, call) for each `self.attr = Name(...)`.""" + for node in ast.walk(cls): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Call): + func = node.value.func + for target in node.targets: + if ( + isinstance(func, ast.Name) + and isinstance(target, ast.Attribute) + and getattr(target.value, "id", None) == "self" + ): + yield target.attr, func.id, node.value + + +def attention_tp_reductions(cls): + """Lines in a class that can turn on use_dp_attention_reduce.""" + found = [] + for fn in ast.walk(cls): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + params = {a.arg for a in fn.args.args + fn.args.kwonlyargs} + for node in ast.walk(fn): + if isinstance(node, ast.keyword) and node.arg == "use_dp_attention_reduce": + if can_be_true(node.value, params): + found.append(node.value.lineno) + elif isinstance(node, ast.Assign) and any( + isinstance(t, ast.Attribute) and t.attr == "use_dp_attention_reduce" + for t in node.targets + ): + if can_be_true(node.value, params): + found.append(node.lineno) + return found + + +class TestFfnExitUsersOweATpSum(CustomTestCase): + """Under attention DP the next layer's input completes a deferred FFN sum + with an all-reduce over TP. An FFN whose linear layers reduce over the + attention-TP group only (use_dp_attention_reduce) owes a different sum, so + no FFN of a layer that uses ffn_exit may turn it on.""" + + def test_no_ffn_behind_ffn_exit_reduces_over_attention_tp(self): + index = SourceIndex() + layers, ffn_classes, offenders = [], set(), [] + for module, path in sorted(index.paths.items()): + source = path.read_text() + if ( + not module.startswith("sglang.srt.models.") + or ".ffn_exit(" not in source + ): + continue + for cls in index.tree(module).body: + if not isinstance(cls, ast.ClassDef): + continue + if ".ffn_exit(" not in ast.get_source_segment(source, cls): + continue + layers.append(f"{module}.{cls.name}") + todo = [] + for attr, name, call in submodule_constructions(cls): + if attr not in FFN_ATTRS: + continue + todo.append((module, name)) + offenders += [ + f"{module}.{cls.name}:{kw.value.lineno}" + for kw in call.keywords + if kw.arg == "use_dp_attention_reduce" + and can_be_true(kw.value, set()) + ] + # Everything an FFN class builds is part of the FFN. + while todo: + resolved = index.resolve(*todo.pop()) + if resolved is None: + continue + where, ffn = resolved + if (where, ffn.name) in ffn_classes: + continue + ffn_classes.add((where, ffn.name)) + offenders += [ + f"{where}.{ffn.name}:{line}" + for line in attention_tp_reductions(ffn) + ] + todo += [(where, n) for _, n, _ in submodule_constructions(ffn)] + self.assertTrue(layers and ffn_classes, "no FFN behind ffn_exit found") + self.assertEqual(sorted(set(offenders)), []) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py index d14d2654f6..3712e177f3 100644 --- a/test/registered/unit/layers/test_layer_communicator_fusion_gate.py +++ b/test/registered/unit/layers/test_layer_communicator_fusion_gate.py @@ -13,7 +13,7 @@ from sglang.srt.layers.moe import ( post_experts_all_reduce, ) from sglang.srt.layers.moe import utils as moe_utils -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_forward, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -298,14 +298,25 @@ class TestDeferFfnReduction(CustomTestCase): shared_expert_tp1=False, lora=False, tp_group=True, + global_tokens=8, + pair_fn=comm.CommunicateSummableTensorPairFn._scatter_hidden_states, + step=comm._redistribute_output, + sp_active=False, ): communicator = _fake_communicator() communicator.is_last_layer = is_last_layer communicator.should_use_reduce_scatter = lambda forward_batch: reduce_scatter + communicator._communicate_summable_tensor_pair_fn = pair_fn + communicator._sp_variant = object() if sp_active else None + communicator.allow_reduce_scatter = True + communicator.layer_scatter_modes.is_layer_sparse = True forward_batch = types.SimpleNamespace( - input_ids=types.SimpleNamespace(shape=(batch_size,)) + input_ids=types.SimpleNamespace(shape=(batch_size,)), + global_dp_buffer_len=global_tokens, ) with ( + patch.object(comm, "_output_to_local_tokens_step", return_value=step), + get_forward().scoped(sp_active=sp_active), patch.object(comm, "is_enable_moe_cp_allgather", return_value=False), patch.object(comm, "apply_flashinfer_allreduce_fusion", return_value=fused), patch.object(comm, "_use_aiter", False), @@ -359,7 +370,6 @@ class TestDeferFfnReduction(CustomTestCase): self, ): for condition in ( - dict(dp_attention=True), dict(a2a_none=False), dict(output_complete=True), dict(quant_communications=True), @@ -374,6 +384,38 @@ class TestDeferFfnReduction(CustomTestCase): self.assertFalse(self._should_defer(**condition)) +class TestDeferFfnReductionUnderAttentionDp(CustomTestCase): + """Under attention DP the next layer's input runs the all-reduce and then the + scatter back to this rank's tokens that postprocess would have run.""" + + _should_defer = TestDeferFfnReduction._should_defer + + def test_defers_when_postprocess_would_only_scatter(self): + self.assertTrue(self._should_defer(dp_attention=True)) + + def test_every_rank_decides_from_all_ranks_tokens(self): + self.assertTrue(self._should_defer(dp_attention=True, batch_size=0)) + self.assertFalse(self._should_defer(dp_attention=True, global_tokens=0)) + + def test_keeps_what_the_next_layer_input_cannot_run(self): + for name, condition in ( + ("reduce-scatter", dict(step=comm._reduce_and_redistribute_output_varlen)), + ( + "MAX_LEN reduce-scatter", + dict(step=comm._reduce_and_redistribute_output_max_len), + ), + ( + "other layout change", + dict( + pair_fn=comm.CommunicateSummableTensorPairFn._scatter_hidden_states_moe + ), + ), + ("LayerNorm SP", dict(sp_active=True)), + ): + with self.subTest(name): + self.assertFalse(self._should_defer(dp_attention=True, **condition)) + + class TestDeferredReductionGroup(CustomTestCase): """The unfused completion runs on the same group object the FFN would have reduced over, or the FFN keeps its reduction.""" diff --git a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py index 5cdaa5ecd7..36a9535b74 100644 --- a/test/registered/unit/layers/test_postprocess_reduce_scatterv.py +++ b/test/registered/unit/layers/test_postprocess_reduce_scatterv.py @@ -151,6 +151,38 @@ class TestPostprocessReduceScatterv(CustomTestCase): self.assertIs(seen["is_layer_sparse"], True) +class TestOutputToLocalTokensStep(CustomTestCase): + """The step that brings a FULL-layout output back to this rank's tokens.""" + + def step(self, *, varlen, max_len, tiles, allow, sparse): + forward_batch = types.SimpleNamespace( + dp_padding_mode=types.SimpleNamespace(is_max_len=lambda: max_len) + ) + with ( + patch.object(comm, "should_use_dp_reduce_scatterv", return_value=varlen), + patch.object(comm, "can_use_dp_reduce_scatter", return_value=tiles), + ): + return comm._output_to_local_tokens_step( + forward_batch, allow_reduce_scatter=allow, is_layer_sparse=sparse + ) + + def test_every_condition(self): + for varlen, max_len, tiles, allow, sparse in itertools.product( + (False, True), repeat=5 + ): + if varlen and (allow or sparse): + expected = comm._reduce_and_redistribute_output_varlen + elif allow and max_len and tiles: + expected = comm._reduce_and_redistribute_output_max_len + else: + expected = comm._redistribute_output + case = dict( + varlen=varlen, max_len=max_len, tiles=tiles, allow=allow, sparse=sparse + ) + with self.subTest(**case): + self.assertIs(self.step(**case), expected) + + class TestLongcatNextnReducesItsMlp(CustomTestCase): """Its dense layer allows no reduce-scatter, so postprocess never sums the MLP output; the MLP must all-reduce it itself.""" diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index e282d82b4c..7e63d29d0c 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -168,6 +168,30 @@ class TestPrepareAttnSteps(CustomTestCase): self.assertEqual(all_reduce.call_count, 1) self.assertEqual(norm.calls, ["norm"]) + def test_an_owed_reduce_scatter_reaches_local_tokens_before_the_norm(self): + for local_rows in (1, 0): + with ( + self.subTest(local_rows=local_rows), + platform(fusion=True) as (_, all_reduce), + ): + norm = Norm() + partial = torch.ones(3, 4) + local = torch.full((local_rows, 4), 7.0) + step = MagicMock(return_value=local) + hidden_states, residual = communicator(norm).prepare_attn( + comm.UnreducedOutput(partial, reduce_and_redistribute=step), + torch.zeros(local_rows, 4), + None, + ) + step.assert_called_once_with(partial) + self.assertEqual(all_reduce.call_count, 0) + if local_rows: + self.assertEqual(norm.calls, ["norm"]) + torch.testing.assert_close(hidden_states, local * 2) + else: + self.assertEqual(norm.calls, []) + self.assertIs(residual, local) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 7edea1bd82..9a9871b3ab 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -46,6 +46,8 @@ class DeferringLayer(nn.Module): lambda batch: defer ) self.layer_communicator.should_defer_ffn_reduction = lambda batch: defer + self.layer_communicator._reduce_scatter_for_next_layer = lambda batch: None + self.layer_communicator._scatter_for_next_layer = lambda batch: None self.layer_communicator.should_use_reduce_scatter = lambda batch: False self.layer_communicator.postprocess_layer = lambda hidden, residual, batch: ( all_reduce(hidden),