diff --git a/python/sglang/srt/layers/layer_boundary/adapters/attention.py b/python/sglang/srt/layers/layer_boundary/adapters/attention.py index 5496aa58b1..5f504dc31d 100644 --- a/python/sglang/srt/layers/layer_boundary/adapters/attention.py +++ b/python/sglang/srt/layers/layer_boundary/adapters/attention.py @@ -55,7 +55,6 @@ class AttentionInputs: self.hidden_states_local = hidden_states self.forward_batch = forward_batch self.qkv_latent_func = qkv_latent_func - self.hidden_states_ = None self.qkv_latent_ = None # When True, hidden_states_local is already attn_tp-gathered upstream # (e.g. by the input-scattered attention input step for DSA). fetch_* must NOT gather again. @@ -72,14 +71,6 @@ class AttentionInputs: self.qkv_latent_ = tp_gather(self.qkv_latent_, self.forward_batch) return self.qkv_latent_ - def fetch_hidden_states(self): - if self.hidden_states_ is not None: - return self.hidden_states_ - self.hidden_states_ = self.hidden_states_local - if get_attn_tp_context().input_scattered and not self.is_pre_gathered: - self.hidden_states_ = tp_gather(self.hidden_states_, self.forward_batch) - return self.hidden_states_ - class AttnTpContext: def __init__(self): @@ -131,11 +122,6 @@ class AttnTpContext: assert attn_inputs is not None return attn_inputs.fetch_qkv_latent() - def fetch_hidden_states(self): - attn_inputs = get_forward().attn_inputs - assert attn_inputs is not None - return attn_inputs.fetch_hidden_states() - def clear_attn_inputs(self) -> None: get_forward().set("attn_inputs", None) diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index fa1afbe5d8..8354b1c466 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -55,7 +55,6 @@ from sglang.srt.layers.layer_boundary.output import OutputTransform from sglang.srt.layers.layer_boundary.prepare import ( _attn_tp_reduce_scatter_update_read, _attn_tp_slice_update_read, - _dispatch_by_update, _dp_gather_sum_read, _move_before_read, _reduce_update_read, @@ -116,6 +115,26 @@ class ExitMove(NamedTuple): # Whether output_move also completes the sum the producer leaves. output_move_completes_sum: bool = False returns_over_dp: bool = False + # Whether output_move gathers the rows back over attention TP. + gathers_attn_tp: bool = False + + +class EntryStep(NamedTuple): + """The steps chosen into a stage, with what they do that binding and + capture rely on.""" + + step: Callable + # Moves the input onto the stage's rows after the step, or None. + input_move: Optional[Callable] = None + # input_move gathers over attention TP with the stage's own gather. + input_gather_declared: bool = False + # input_move returns storage of its own on every path it takes, which a + # capture may keep without copying: the boundary's gather does; a stage's + # own gather may return a buffer it reuses. + input_retainable: bool = False + # The kernels the step tries for the sum over attention TP it completes + # without moving the residual. + ffn_fusions: Tuple[FfnInputFusion, ...] = () def input_rows(edge: EdgeContract) -> Layout: @@ -203,8 +222,6 @@ def bind_entry( Returns: An EntryPath whose prepare accepts the actual update from the stream. - Multiple update capabilities select among preconstructed paths; a single - capability needs no runtime dispatcher. """ if edge.produced.transform is not None: # An attention's transform, run once its sum is complete. @@ -216,41 +233,20 @@ def bind_entry( ), residual_joins_sum=False, ) - capabilities = edge.arriving_plain_add - if not capabilities: + is_plain_add = edge.arriving_plain_add + if is_plain_add is None: if edge.produced.update is None: - raise ValueError("an arrival must declare its update capabilities") - capabilities = (edge.produced.update.is_plain_add,) - gather_input = ( - attn_tp_gather_input - if attn_tp_gather is None - else partial(attn_tp_gather_input, gather=attn_tp_gather) - ) - paths = { - capability: _bind_entry_path( - edge, - is_plain_add=capability, - fusions=fusions, - carried_fusions=carried_fusions, - cp_moves=cp_moves, - enters_stack=enters_stack, - attn_input_adapter=attn_input_adapter, - gather_input=gather_input, - ) - for capability in capabilities - } - first = next(iter(paths.values())) - if len(paths) == 1: - return first - if any(path.input_move != first.input_move for path in paths.values()): - raise NotImplementedError("update capabilities require different input moves") - return msgspec.structs.replace( - first, - preserves_residual=None, - prepare=partial( - _dispatch_by_update, - paths={capability: path.prepare for capability, path in paths.items()}, - ), + raise ValueError("an arrival must declare its update capability") + is_plain_add = edge.produced.update.is_plain_add + return _bind_entry_path( + edge, + is_plain_add=is_plain_add, + fusions=fusions, + carried_fusions=carried_fusions, + cp_moves=cp_moves, + enters_stack=enters_stack, + attn_input_adapter=attn_input_adapter, + attn_tp_gather=attn_tp_gather, ) @@ -263,7 +259,7 @@ def _bind_entry_path( cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, attn_input_adapter: Optional[Callable] = None, - gather_input: Callable = attn_tp_gather_input, + attn_tp_gather: Optional[Callable] = None, ) -> EntryPath: """The consumer's half of ``edge``, chosen from the edge's declarations and the producer's update: what a value carries for a batch (a sum or a handoff @@ -277,7 +273,7 @@ def _bind_entry_path( producer chose for a batch.""" # A fused kernel runs the add and the norm itself. plain = is_plain_add and edge.need.read.is_plain_norm - step, input_move = _select_entry_step( + entry = _select_entry_step( edge.produced, residual=edge.residual, residual_to=edge.residual_to, @@ -287,12 +283,12 @@ def _bind_entry_path( residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, - gather_input=gather_input, + attn_tp_gather=attn_tp_gather, arrives_written=edge.arrives_written, ) completed_step = None if edge.produced.always_partial: - completed_step, _ = _select_entry_step( + completed_step = _select_entry_step( msgspec.structs.replace(edge.produced, always_partial=False), residual=edge.residual, residual_to=edge.residual_to, @@ -302,12 +298,12 @@ def _bind_entry_path( residual_joins_sum=False, cp_moves=cp_moves, enters_stack=enters_stack, - ) + ).step # A written stream entering the first physical layer must not run enter # again (e.g. MHC expansion). Both alternatives are bound at construction. written_step = None if enters_stack: - written_step, _ = _select_entry_step( + written_step = _select_entry_step( edge.produced, residual=edge.residual, residual_to=edge.residual_to, @@ -317,28 +313,22 @@ def _bind_entry_path( residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=False, - ) + ).step preserves_residual = None if ( edge.produced.always_partial and edge.produced.group is SumGroup.ATTN_TP - and isinstance(step, partial) - and step.func is _reduce_update_read - and not step.keywords["gathers_residual"] + and entry.ffn_fusions ): - selected = step.keywords["fusions"] - if selected: - # Only the first reachable candidate can certify ownership; an - # earlier custom candidate could otherwise mutate the residual. - preserves_residual = next( - f.preserves_residual for f in fusions if f.run == selected[0] - ) + # Only the first reachable candidate can certify ownership; an + # earlier custom candidate could otherwise mutate the residual. + preserves_residual = entry.ffn_fusions[0].preserves_residual declared_sum = edge.produced.group if edge.produced.always_partial else None capture_move, capture_move_allocates = _capture_move(edge) return EntryPath( prepare=partial( _run_entry, - step=step, + step=entry.step, is_plain_add=is_plain_add, carried_fusions=carried_fusions if plain else (), expected_sum=declared_sum, @@ -346,7 +336,9 @@ def _bind_entry_path( written_step=written_step, ), input_rows=input_rows(edge), - input_move=input_move, + input_move=entry.input_move, + input_gather_declared=entry.input_gather_declared, + input_retainable=entry.input_retainable, attn_input_adapter=attn_input_adapter, capture_move=capture_move, capture_move_allocates=capture_move_allocates, @@ -420,9 +412,9 @@ def _select_entry_step( residual_joins_sum: bool, cp_moves: Optional[CpMoves], enters_stack: bool, - gather_input: Callable = attn_tp_gather_input, + attn_tp_gather: Optional[Callable] = None, arrives_written: bool = False, -) -> Tuple[Callable, Optional[Callable]]: +) -> EntryStep: """The steps into a stage for a value that is complete or owes the sum its producer always leaves, the fused kernels they try first, and the move after them: complete that sum, move the residual to the rows it has while @@ -454,7 +446,7 @@ def _select_entry_step( local = msgspec.structs.replace( produced, layout=residual_to, group=None, always_partial=False ) - read_local, input_move = _select_entry_step( + inner = _select_entry_step( local, residual=residual, residual_to=residual_to, @@ -465,11 +457,15 @@ def _select_entry_step( cp_moves=cp_moves, enters_stack=enters_stack, ) - return partial( - _move_before_read, - move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back, - read=read_local, - ), input_move + return inner._replace( + step=partial( + _move_before_read, + move=cp_moves.reduce_scatter + if owes is not None + else cp_moves.take_back, + read=inner.step, + ) + ) if sliced: # Each attention-TP rank takes its own slice: the reduce-scatter # completes the attention-TP sum and slices in one collective; a @@ -489,9 +485,8 @@ def _select_entry_step( ) else: step = _attn_tp_slice_update_read - return ( - partial(step, scatters_residual=residual != residual_to, read=read), - None, + return EntryStep( + partial(step, scatters_residual=residual != residual_to, read=read) ) if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_TP}: if residual == residual_to: @@ -499,13 +494,12 @@ def _select_entry_step( # rows around it (MHC on an input-scattered batch). if gathered or owes not in (None, SumGroup.ATTN_TP): raise NotImplementedError(f"{produced=} {residual=} {need=}") - return ( + return EntryStep( partial( _tp_reduce_scatter_update_read_gather, read=read, reduces=owes is not None, - ), - None, + ) ) # A reduce-scatter completes the TP sum onto each rank's slice, which # the stage takes: its group is the TP group without attention DP or CP. @@ -516,20 +510,19 @@ def _select_entry_step( or residual_to.sharded != {TokenAxis.ATTN_TP} ): raise NotImplementedError(f"{produced=} {residual=} {need=}") - return ( + return EntryStep( partial( _update_read, pre_move=tp_reduce_scatter if owes is not None else tp_slice, enters_stack=enters_stack, read=read, - ), - None, + ) ) if gathered == {TokenAxis.ATTN_CP}: # Each CP rank completes its own chunk, then the CP moves gather them. if cp_moves is None: raise NotImplementedError(f"{produced=} {need=}") - on_chunk, _ = _select_entry_step( + on_chunk = _select_entry_step( produced, residual=residual, residual_to=residual_to, @@ -539,8 +532,8 @@ def _select_entry_step( residual_joins_sum=residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, - ) - return (partial(cp_moves.gather, gather=on_chunk), None) + ).step + return EntryStep(partial(cp_moves.gather, gather=on_chunk)) if gathered == {TokenAxis.ATTN_TP}: # A complete input on each rank's slice, gathered over attention TP # once it is read, unless one of the read's own kernels gathers it. @@ -550,7 +543,7 @@ def _select_entry_step( or residual_to != produced.layout ): raise NotImplementedError(f"{produced=} {residual=} {need=}") - return ( + return EntryStep( partial( _update_read, pre_move=None, @@ -562,7 +555,13 @@ def _select_entry_step( else () ), ), - gather_input, + input_move=( + attn_tp_gather_input + if attn_tp_gather is None + else partial(attn_tp_gather_input, gather=attn_tp_gather) + ), + input_gather_declared=attn_tp_gather is not None, + input_retainable=attn_tp_gather is None, ) if ( residual_to != produced.layout @@ -581,31 +580,29 @@ def _select_entry_step( if not gathered: if owes is None: if gathers_residual: - return ( + return EntryStep( partial( _reduce_update_read, gathers_residual=True, fusions=(), group=None, read=read, - ), - None, + ) ) - return ( + return EntryStep( partial( _update_read, pre_move=None, enters_stack=enters_stack, read=read, - ), - None, + ) ) if gathers_residual and residual_joins_sum: # Each rank adds its slice of the residual into its share of the # sum, so the all-reduce also brings the residual back to every row. if owes is not SumGroup.ATTN_TP or not is_plain_add: raise NotImplementedError(f"{produced=} {residual=} {need=}") - return (partial(_tp_sum_with_residual_read, read=read), None) + return EntryStep(partial(_tp_sum_with_residual_read, read=read)) # A sum over TP completes only on rows every TP rank holds: the TP group # spans attention DP and CP. if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and ( @@ -614,7 +611,7 @@ def _select_entry_step( raise NotImplementedError(f"{produced=} {need=}") fused = tuple(f for f in fusions if f.completes is owes) read_fused = _read_fusions(read, owes, is_plain_add, scatters=False) - return ( + return EntryStep( partial( _reduce_update_read, gathers_residual=gathers_residual, @@ -623,7 +620,7 @@ def _select_entry_step( group=owes, read=read, ), - None, + ffn_fusions=() if gathers_residual else fused, ) if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP): raise NotImplementedError(f"{produced=} {need=}") @@ -640,16 +637,15 @@ def _select_entry_step( and is_plain_add and read.is_plain_norm ): - return ( + return EntryStep( partial( _dp_gather_sum_read, gathers_residual=gathers_residual, places_cp_shards=places_cp_shards, read=read, - ), - None, + ) ) - return ( + return EntryStep( partial( _reduce_update_read_dp_gather, gathers_residual=gathers_residual, @@ -657,8 +653,7 @@ def _select_entry_step( group=owes, places_cp_shards=places_cp_shards, read=read, - ), - None, + ) ) @@ -687,7 +682,8 @@ def _select_exit_move( update_attn_tp_gather_output, update=update, gather=attn_tp_gather, - ) + ), + gathers_attn_tp=True, ) raise NotImplementedError(f"{produced=} {residual=} {to=}") returned = residual.sharded - produced.layout.sharded diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index f426ec14a6..080377f960 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -247,6 +247,7 @@ class StagePlan: output=edges.outgoing.produced, output_move=out.output_move, output_move_completes_sum=out.output_move_completes_sum, + output_gathers_attn_tp=out.gathers_attn_tp, returns_over_dp=out.returns_over_dp, writes_at_handoff=writes_at_handoff, ) diff --git a/python/sglang/srt/layers/layer_boundary/contracts.py b/python/sglang/srt/layers/layer_boundary/contracts.py index 6ca117618c..005cafb28a 100644 --- a/python/sglang/srt/layers/layer_boundary/contracts.py +++ b/python/sglang/srt/layers/layer_boundary/contracts.py @@ -140,9 +140,9 @@ class EdgeContract(msgspec.Struct, frozen=True): residual_to: Residual layout after the boundary. residual_joins_sum: Whether one rank may add the residual into a partial before reduction; valid only for an eligible plain-add update. - arriving_plain_add: Allowed values of ResidualUpdate.is_plain_add for - arriving contributions. Empty means use produced.update's capability. - The actual update object travels with the residual stream. + arriving_plain_add: ResidualUpdate.is_plain_add of the arriving + contribution, whose update object travels with the residual stream; + None means use produced.update's. arrives_written: Whether the producer applies its update at its exit, so the stream arrives written with no residual add pending. """ @@ -154,8 +154,8 @@ class EdgeContract(msgspec.Struct, frozen=True): # Whether the residual is added into one rank's share of the produced sum # before that sum completes, instead of after it. residual_joins_sum: bool = False - # Capabilities allowed to arrive from another layer, not its update object. - arriving_plain_add: Tuple[bool, ...] = () + # The capability arriving from another layer, not its update object. + arriving_plain_add: Optional[bool] = None arrives_written: bool = False @@ -220,6 +220,11 @@ class EntryPath(msgspec.Struct, frozen=True): applies the producer update and performs the consumer read. input_rows: Layout handed to compute after preparation and input_move. input_move: Optional movement after prepare, before compute takes the input. + input_gather_declared: Whether input_move gathers over attention TP + with the stage's own gather. + input_retainable: Whether a capture may keep input_move's output + without copying: it is storage of its own on every path the move + takes. attn_input_adapter: Optional callable(input, forward_batch, qkv_latent_func) that adapts already-placed input for attention. capture_move: Optional movement of the updated residual back onto the @@ -243,6 +248,8 @@ class EntryPath(msgspec.Struct, frozen=True): # Moves the input onto the stage's rows after prepare, when prepare does # not: (hidden_states, forward_batch) -> hidden_states. input_move: Optional[Callable] = None + input_gather_declared: bool = False + input_retainable: bool = False # Hands the stage its input once it is on its rows: # (hidden_states, forward_batch, qkv_latent_func) -> hidden_states. attn_input_adapter: Optional[Callable] = None @@ -264,6 +271,8 @@ class StagePath(msgspec.Struct, frozen=True): output_move: Fixed output transport, or None when absent or chosen per batch by the attention-DP exit path. output_move_completes_sum: Whether that move also reduces the output. + output_gathers_attn_tp: Whether that move gathers the rows back over + attention TP. returns_over_dp: Whether output uses batch-dependent attention-DP transport. writes_at_handoff: Whether the exit completes the output and writes it into the residual, for an FFN that hands off to another pipeline rank. @@ -274,6 +283,7 @@ class StagePath(msgspec.Struct, frozen=True): # None means no fixed move. returns_over_dp selects batch-dependent DP transport. output_move: Optional[Callable] output_move_completes_sum: bool = False + output_gathers_attn_tp: bool = False returns_over_dp: bool = False writes_at_handoff: bool = False diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index fed64cf858..388aec541a 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -44,10 +44,6 @@ from sglang.srt.layers.layer_boundary.layout import ( moe_gathers_over_moe_cp, token_axis_sizes, ) -from sglang.srt.layers.layer_boundary.ops import ( - attn_tp_gather_input, - update_attn_tp_gather_output, -) from sglang.srt.layers.layer_boundary.output import OutputTransform from sglang.srt.layers.layer_boundary.residual import ResidualReadout, ResidualUpdate from sglang.srt.layers.layer_boundary.residual.add_norm import ( @@ -526,7 +522,7 @@ def _connect(producer, consumer, *, residual_from=None): update=None, ) residual = rows - capabilities = (True,) + arriving_plain_add = True else: decl, during, returned = _resolve_stage(before, variant, following=after) if during is None: @@ -550,7 +546,7 @@ def _connect(producer, consumer, *, residual_from=None): before.kind is StageKind.ATTENTION and before.reduction is ProducerReduction.ALWAYS_PARTIAL ): - arrived, residual, capabilities = decl.output, during, () + arrived, residual, arriving_plain_add = decl.output, during, None else: owes = ( variant is BatchVariant.INPUT_SCATTERED @@ -574,7 +570,8 @@ def _connect(producer, consumer, *, residual_from=None): else False, update=None, ) - residual, capabilities = returned, (before.update.is_plain_add,) + residual = returned + arriving_plain_add = before.update.is_plain_add written = before.update.applied_at_exit if after is None: continue @@ -606,7 +603,7 @@ def _connect(producer, consumer, *, residual_from=None): residual, during, residual_joins_sum=joins, - arriving_plain_add=capabilities, + arriving_plain_add=arriving_plain_add, arrives_written=written, ) entries[variant] = edge @@ -638,7 +635,7 @@ def _fork_input(prepared, consumer): declaration.input, source.residual_to, during, - arriving_plain_add=(True,), + arriving_plain_add=True, ) return StageConnection(prepared.consumer, consumer, {}, entries) @@ -897,13 +894,13 @@ def _check_declared_gathers(appends, bound, *, remote_producer): ] for (origin, consumer), producer in zip(line, [None, *(b for _, b in line)]): if consumer.declaration.attn_tp_gather is None or any( - _gathers_input(path) for path in consumer.plan.paths.values() + path.entry.input_gather_declared for path in consumer.plan.paths.values() ): continue if producer is None and remote_producer: continue if producer is not None and any( - _gathers_output(path) for path in producer.plan.paths.values() + path.output_gathers_attn_tp for path in producer.plan.paths.values() ): continue error = ValueError( @@ -914,14 +911,6 @@ def _check_declared_gathers(appends, bound, *, remote_producer): raise error -def _gathers_input(path): - return getattr(path.entry.input_move, "func", None) is attn_tp_gather_input - - -def _gathers_output(path): - return getattr(path.output_move, "func", None) is update_attn_tp_gather_output - - def _ended(declaration, final_read): """The stack's last stage as the final read takes it: an FFN on this rank's attention-TP slice leaves its output there for a read that reads diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 66c6c8b281..d0268b381c 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -263,21 +263,6 @@ def _attn_input_scattered( return hidden_states -def _dispatch_by_update( - hidden_states, residual, forward_batch, norm, *, paths, update=PLAIN_ADD, **call -): - if residual is None: - # A missing residual cannot join a partial sum. The non-plain path - # performs init_residual/read without that optimization. - prepare = paths[False] - else: - try: - prepare = paths[update.is_plain_add] - except KeyError: - raise RuntimeError("producer update has no bound input path") from None - return prepare(hidden_states, residual, forward_batch, norm, update=update, **call) - - def _run_entry( hidden_states: Union[torch.Tensor, UnreducedOutput, DeferredFinalize], residual: Optional[torch.Tensor], diff --git a/python/sglang/srt/layers/layer_boundary/stage.py b/python/sglang/srt/layers/layer_boundary/stage.py index 0a5bfbdd25..b7aa085ba2 100644 --- a/python/sglang/srt/layers/layer_boundary/stage.py +++ b/python/sglang/srt/layers/layer_boundary/stage.py @@ -31,7 +31,7 @@ from sglang.srt.layers.layer_boundary.contracts import ( StagePath, ) from sglang.srt.layers.layer_boundary.layout import SumGroup -from sglang.srt.layers.layer_boundary.ops import attn_tp_gather_input, sum_output +from sglang.srt.layers.layer_boundary.ops import sum_output from sglang.srt.layers.layer_boundary.residual.access import buffer, from_pp from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, @@ -395,11 +395,10 @@ class StageBoundary: forward_batch=forward_batch, ) ) - # Input gathers allocate fresh DP buffers (see attn_tp_gather_input). # Without a gather this is the mutable residual, so retention copies. capture_gathered.capture( gathered_last_layer_output, - owned=move is attn_tp_gather_input + owned=entry.input_retainable or ( move is None and entry.capture_preserves_residual is not None diff --git a/test/registered/unit/layer_boundary/test_aux_hidden_states.py b/test/registered/unit/layer_boundary/test_aux_hidden_states.py index f174945baf..5a2b33f933 100644 --- a/test/registered/unit/layer_boundary/test_aux_hidden_states.py +++ b/test/registered/unit/layer_boundary/test_aux_hidden_states.py @@ -97,6 +97,7 @@ class TestAuxStorage(CustomTestCase): ) stub_stage(boundary, StageKind.ATTENTION).entry = lambda _: SimpleNamespace( input_move=move, + input_retainable=False, capture_move=None, capture_move_allocates=False, capture_preserves_residual=None, diff --git a/test/registered/unit/layer_boundary/test_boundary_edges.py b/test/registered/unit/layer_boundary/test_boundary_edges.py index 3651033cbd..fdd7fb8448 100644 --- a/test/registered/unit/layer_boundary/test_boundary_edges.py +++ b/test/registered/unit/layer_boundary/test_boundary_edges.py @@ -3,7 +3,6 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch -import msgspec import torch from sglang.srt.layers import layer_boundary as comm @@ -224,46 +223,6 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): self.assertTrue(other.keywords["reduces_attention_tp"]) self.assertNotIn("update", other.keywords) - def test_capabilities_prebind_both_orders_without_capturing_an_update(self): - edge = msgspec.structs.replace( - self.edge(None), arriving_plain_add=(True, False) - ) - boundary = bind_entry(edge) - paths = boundary.prepare.keywords["paths"] - self.assertEqual(set(paths), {True, False}) - self.assertIs(paths[True].keywords["step"].func, comm_ops._dp_gather_sum_read) - self.assertIs( - paths[False].keywords["step"].func, comm_ops._reduce_update_read_dp_gather - ) - for path in paths.values(): - self.assertNotIn("update", path.keywords["step"].keywords) - - def test_actual_update_selects_a_prebound_path(self): - full = rows(sizes(tp=1)) - edge = EdgeContract( - OutputContract(full, update=None), - InputContract(full), - full, - full, - arriving_plain_add=(True, False), - ) - boundary = bind_entry(edge) - - def norm(value, residual=None): - return value if residual is None else (value + residual, value + residual) - - for update, expected in ((comm.PLAIN_ADD, 4.0), (_WrittenIn(), 5.0)): - with self.subTest(plain=update.is_plain_add): - hidden, residual = boundary.prepare( - torch.ones(2, 4), - torch.full((2, 4), 3.0), - None, - norm, - update=update, - ) - torch.testing.assert_close(hidden, torch.full((2, 4), expected)) - torch.testing.assert_close(residual, hidden) - def test_fused_kernels_take_only_a_plain_add(self): full = rows(sizes(tp=2)) fused = comm.FfnInputFusion(completes=SumGroup.ATTN_TP, run=MagicMock()) diff --git a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py index 1d35a654e7..906e4f9100 100644 --- a/test/registered/unit/layer_boundary/test_boundary_output_contracts.py +++ b/test/registered/unit/layer_boundary/test_boundary_output_contracts.py @@ -17,7 +17,6 @@ from sglang.srt.layers.layer_boundary import stage as stages from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.layers.layer_boundary.fusions.cutedsl import CuteDSLFusion from sglang.srt.layers.layer_boundary.layout import SumGroup -from sglang.srt.layers.layer_boundary.prepare import _dispatch_by_update from sglang.srt.layers.layer_boundary.residual import batch from sglang.srt.layers.layer_boundary.residual.add_norm import REPLACE_AT_EXIT from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream @@ -134,17 +133,6 @@ class TestBoundaryIntegrations(unittest.TestCase): decision = ffn.exit(fb) self.assertEqual(decision.defer_moe_finalize, accepted) - def test_absent_residual_uses_non_partial_update_path(self): - plain = Mock(side_effect=AssertionError("cannot add a missing residual")) - generic = Mock(return_value=("input", None)) - self.assertEqual( - _dispatch_by_update( - None, None, None, None, paths={True: plain, False: generic} - ), - ("input", None), - ) - plain.assert_not_called() - def test_lora_or_shared_tp1_defers_only_when_fusion_is_eligible(self): group = object() residual = torch.ones(2, 4) diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 77d81d8151..3670806aa7 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -730,7 +730,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): TokenAxis.ATTN_CP: 1, TokenAxis.ATTN_TP: 2, } - steps, _ = comm_boundary._select_entry_step( + steps, *_ = comm_boundary._select_entry_step( produced, residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), @@ -813,7 +813,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): TokenAxis.ATTN_TP: 2, } attention = comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes) - steps, _ = comm_boundary._select_entry_step( + steps, *_ = comm_boundary._select_entry_step( comm.OutputContract(attention, group=SumGroup.ATTN_TP, always_partial=True), residual=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), residual_to=comm.Layout.sharded_over(TokenAxis.ATTN_DP, axis_sizes=sizes), @@ -901,7 +901,7 @@ class TestInputScatteredAttention(CustomTestCase): (False, comm_ops._reduce_update_read), ): with self.subTest(residual_joins_sum=joins): - steps, _ = comm_boundary._select_entry_step( + steps, *_ = comm_boundary._select_entry_step( owed, residual=local, residual_to=attention, @@ -920,7 +920,7 @@ class TestInputScatteredAttention(CustomTestCase): sizes = self.SIZES attention = comm.Layout.sharded_over(axis_sizes=sizes) local = comm.Layout(frozenset({TokenAxis.ATTN_TP})) - step, _ = comm_boundary._select_entry_step( + step, *_ = comm_boundary._select_entry_step( comm.OutputContract(attention), residual=attention, residual_to=local,