From a956aa192abe7efb5e4708153ba31d33e04bc5b7 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:42:00 -0700 Subject: [PATCH] [Refactor] Bind a layer stack in one pass over each stage's data flow (#43457) --- .../srt/layers/layer_boundary/construction.py | 40 +- .../srt/layers/layer_boundary/factories.py | 682 +++++++++--------- .../test_independent_stage_construction.py | 5 +- .../layer_boundary/test_nemotron_h_stages.py | 26 +- 4 files changed, 373 insertions(+), 380 deletions(-) diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 679516d59a..06899642ab 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -17,12 +17,15 @@ from dataclasses import dataclass from functools import cached_property from typing import Callable, Optional +import msgspec + from sglang.srt.layers.layer_boundary.adapters.attention import get_attn_tp_context from sglang.srt.layers.layer_boundary.boundary import ( ExitMove, _cp_moves, bind_entry, bind_exit, + input_rows, ) from sglang.srt.layers.layer_boundary.contracts import ( BatchVariant, @@ -127,6 +130,8 @@ class StagePlan: fusions=None, attn_tp_gather=None, exit_gather=None, + next_input_rows=None, + capture_preserves_residual=None, ): self.norm = norm self.edges = dict(variants) @@ -146,7 +151,7 @@ class StagePlan: self._publish_lora_layout = get_parallel().attn_dp_enabled and bool( get_lora().enable_lora ) - self._next_input_rows = None + self._next_input_rows = next_input_rows carried = ( attn_input_fusions(self, next(iter(self.edges.values())).incoming.need.read) if kind is StageKind.ATTENTION @@ -172,6 +177,11 @@ class StagePlan: attn_input_adapter=edges.attn_input_adapter, attn_tp_gather=attn_tp_gather, ) + keeps = (capture_preserves_residual or {}).get(variant) + if keeps is not None: + entry = msgspec.structs.replace( + entry, capture_preserves_residual=keeps + ) out = ( ExitMove() if finishes_directly @@ -251,7 +261,16 @@ def _bound_for(bound, variant): ) from None -def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **options): +def _bind_stage( + declaration, + norm, + incoming, + outgoing, + *, + final_read=None, + capture_preserves_residual=None, + **options, +): if incoming.consumer != declaration or outgoing.producer != declaration: raise ValueError("connections do not match the stage declaration") if incoming.entries.keys() != outgoing.exits.keys(): @@ -307,15 +326,14 @@ def _bind_stage(declaration, norm, incoming, outgoing, *, final_read=None, **opt ), finishes_directly=declaration.kind is StageKind.ATTENTION and declaration.reduction is ProducerReduction.ALWAYS_PARTIAL, - **options, - ) - if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None: # Layout eligibility comes from the connected consumer, not a mutable # link to its execution plan. Kernel binding remains consumer-owned. - from sglang.srt.layers.layer_boundary.boundary import input_rows - - plan._next_input_rows = { - v: input_rows(edge) for v, edge in outgoing.entries.items() - } - + next_input_rows=( + {v: input_rows(edge) for v, edge in outgoing.entries.items()} + if declaration.kind is StageKind.ATTENTION and outgoing.consumer is not None + else None + ), + capture_preserves_residual=capture_preserves_residual, + **options, + ) return StageBoundary(plan, declaration=declaration) diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index ddf460f86e..15e2a82e6f 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -7,9 +7,7 @@ import os import sys from dataclasses import dataclass, replace from types import MappingProxyType -from typing import Callable, Mapping, Optional - -import msgspec +from typing import Callable, Mapping, NamedTuple, Optional from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp @@ -520,7 +518,49 @@ def _reject_unsupported(stage): ) -def _resolve_stage(stage, variant, following=None): +class _Arrival(NamedTuple): + """What reaches a stage's entry from its producer, for one batch variant. + ``plain_add`` is None after an attention that always leaves its sum: the + entry receives that update with the output.""" + + produced: OutputContract + residual: Layout + plain_add: Optional[bool] + written: bool + + +class _Flow(NamedTuple): + """One stage's residual data flow for one batch variant.""" + + contract: StageContract + arrival: _Arrival + during: Layout + returned: Layout + + +def _stack_arrival(variant): + """What reaches the first stage of the model's layer stack: its input, on + the attention's rows (this rank's slice of them under sequence + parallelism), owing the TP sum an input-scattered batch leaves.""" + _, attention, local, _ = _row_layouts(variant) + rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention + owes = variant is BatchVariant.INPUT_SCATTERED + return _Arrival( + OutputContract( + rows, + group=SumGroup.TP if owes else None, + always_partial=owes, + update=None, + ), + rows, + True, + False, + ) + + +def _resolve_stage(stage, variant, arrival, following=None): + """``stage``'s data flow for ``variant``, given what arrives at its entry + and the stage after it (None when it ends the stack or a branch).""" axes, attention, local, full = _row_layouts(variant) _reject_unsupported(stage) if stage.kind is StageKind.FFN: @@ -543,7 +583,7 @@ def _resolve_stage(stage, variant, following=None): # The residual's rows during the FFN: this rank's slice for an FFN # on its own rows, else the attention's. returned = residual - return declaration, residual, returned + return _Flow(declaration, arrival, residual, returned) sp = variant is BatchVariant.SEQUENCE_PARALLEL scattered = variant is BatchVariant.INPUT_SCATTERED gathers = ( @@ -589,170 +629,165 @@ def _resolve_stage(stage, variant, following=None): transform=stage.output_transform, ), ) - return declaration, None, attention + # An attention moves no residual: it keeps the rows it arrives on, except + # that one that always leaves its sum completes an input-scattered sum + # it receives onto this rank's slice of them. + during = arrival.residual + if ( + variant is BatchVariant.INPUT_SCATTERED + and arrival.produced.always_partial + and arrival.produced.update is None + and stage.reduction is ProducerReduction.ALWAYS_PARTIAL + ): + during = Layout(during.sharded | {TokenAxis.ATTN_TP}) + return _Flow(declaration, arrival, during, attention) -def _connect(producer, consumer, *, residual_from=None): - """Resolve producer and consumer declarations; None denotes an external endpoint. - - A missing producer is the stack input; a missing consumer is a layer or stack exit - whose next read is bound separately (or the terminal output). - ``residual_from`` names the boundary that placed the producer's residual. - It is needed for attention, whose residual can stay on finer rows than its - compute input. No executable stage participates in this construction. - """ - if producer is None and consumer is None: - raise ValueError("a boundary needs at least one declared side") - before, after = producer, consumer - exits, entries = {}, {} - for variant in _active_variants(): - _, attention, local, _ = _row_layouts(variant) - written = False - if before is None: - rows = local if variant is BatchVariant.SEQUENCE_PARALLEL else attention - owes = variant is BatchVariant.INPUT_SCATTERED - arrived = OutputContract( - rows, - group=SumGroup.TP if owes else None, - always_partial=owes, - update=None, - ) - residual = rows - arriving_plain_add = True - else: - decl, during, returned = _resolve_stage(before, variant, following=after) - if during is None: - if residual_from is not None: - if _detached(residual_from.consumer) != _detached(producer): - raise ValueError("residual source must enter the producer") - during = residual_from.entries[variant].residual_to - elif ( - before.kind is StageKind.ATTENTION - and before.reduction is ProducerReduction.EXIT_SCOPED - ): - during = attention - else: - raise ValueError( - "attention output needs its incoming residual placement" - ) - exits[variant] = EdgeContract( - decl.output, InputContract(returned), during, returned - ) - if ( - before.kind is StageKind.ATTENTION - and before.reduction is ProducerReduction.ALWAYS_PARTIAL - ): - arrived, residual, arriving_plain_add = decl.output, during, None - else: - owes = ( - variant is BatchVariant.INPUT_SCATTERED - and not before.update.applied_at_exit - ) - carries = ( - ( - before.kind is StageKind.ATTENTION - and before.reduction is ProducerReduction.EXIT_SCOPED - ) - or resolve_exit_rows(before.exit_rows) is ExitRows.ATTENTION - ) and (decl.output.always_partial or decl.output.may_defer_to_next) - arrived = OutputContract( - returned, - group=decl.output.group - if carries - else (SumGroup.TP if owes else None), - always_partial=decl.output.always_partial if carries else owes, - may_defer_to_next=decl.output.may_defer_to_next - if carries - else False, - update=None, - ) - residual = returned - arriving_plain_add = before.update.is_plain_add - written = before.update.applied_at_exit - if after is None: - continue - decl, during, _ = _resolve_stage(after, variant) - if after.kind is StageKind.ATTENTION: - during = ( - Layout(residual.sharded | {TokenAxis.ATTN_TP}) - if variant is BatchVariant.INPUT_SCATTERED - and arrived.always_partial - and arrived.update is None - and after.reduction is ProducerReduction.ALWAYS_PARTIAL - else residual - ) - joins = ( +def _arrival(producer, flow, variant): + """What reaches the stage after ``producer`` from it, given its flow.""" + output = flow.contract.output + if ( + producer.kind is StageKind.ATTENTION + and producer.reduction is ProducerReduction.ALWAYS_PARTIAL + ): + # The output owes its sum, and the residual is where it ran. + return _Arrival(output, flow.during, None, False) + if ( + producer.kind is StageKind.ATTENTION + and producer.reduction is ProducerReduction.EXIT_SCOPED + and (output.always_partial or output.may_defer_to_next) + ): + # A mixer's exit leaves the next entry the sum it declares; a sum it + # may carry on arrives with the value, if at all. + owes, group = output.always_partial, output.group + else: + # On an input-scattered batch, the next entry's reduce-scatter + # completes the TP sum of an output not written at its exit. + owes = ( variant is BatchVariant.INPUT_SCATTERED - and arrived.update is not None - and arrived.update.is_plain_add - and after.kind is StageKind.FFN + and not producer.update.applied_at_exit ) - edge = EdgeContract( - arrived, - decl.input, - residual, - during, - residual_joins_sum=joins, - arriving_plain_add=arriving_plain_add, - arrives_written=written, + group = SumGroup.TP + return _Arrival( + OutputContract( + flow.returned, + group=group if owes else None, + always_partial=owes, + update=None, + ), + flow.returned, + producer.update.is_plain_add, + producer.update.applied_at_exit, + ) + + +def _entry_edge(stage, flow, variant): + """The edge into ``stage`` as its entry binds it.""" + arrival = flow.arrival + joins = ( + variant is BatchVariant.INPUT_SCATTERED + and arrival.produced.update is not None + and arrival.produced.update.is_plain_add + and stage.kind is StageKind.FFN + ) + return EdgeContract( + arrival.produced, + flow.contract.input, + arrival.residual, + flow.during, + residual_joins_sum=joins, + arriving_plain_add=arrival.plain_add, + arrives_written=arrival.written, + ) + + +def _exit_edge(stage, flow, entry): + """The edge out of ``stage`` as its exit binds it. An attention that always + leaves its sum has no exit: its edge is its consumer's entry edge, when + there is a consumer.""" + if entry is not None and ( + stage.kind is StageKind.ATTENTION + and stage.reduction is ProducerReduction.ALWAYS_PARTIAL + ): + return entry + return EdgeContract( + flow.contract.output, InputContract(flow.returned), flow.during, flow.returned + ) + + +def _connect_line(line, origins, *, before=None, after=None, arrivals=None): + """The connections along a chain of stages, each resolved once per batch + variant: into ``line[0]``, between each two, and out of ``line[-1]`` to + ``after`` (a stage on the next rank, or None at the end of the stack or a + branch). What arrives at ``line[0]`` is ``arrivals``, else what ``before`` + (a stage on the previous rank, or None for the stack input) hands on; that + rank's own entry is taken as the stack input's. An error resolving a stage + names the append at ``origins[i]`` that declared ``line[i]``, or the one + next to the neighbouring rank's stage.""" + variants = tuple(_active_variants()) + + def resolve(position, stage, arrivals, following=None): + try: + return { + v: _resolve_stage(stage, v, arrivals[v], following) for v in variants + } + except Exception as error: + _note_origin(error, origins[position]) + raise + + if arrivals is None: + arrivals = {v: _stack_arrival(v) for v in variants} + if before is not None: + flow = resolve(0, before, arrivals, line[0]) + arrivals = {v: _arrival(before, flow[v], v) for v in variants} + flows = [] + for position, stage in enumerate(line): + following = line[position + 1] if position + 1 < len(line) else after + flow = resolve(position, stage, arrivals, following) + flows.append(flow) + arrivals = {v: _arrival(stage, flow[v], v) for v in variants} + if after is not None: + flows.append(resolve(len(line) - 1, after, arrivals)) + entries = [ + {v: _entry_edge(stage, flow[v], v) for v in variants} + for stage, flow in zip([*line, after], flows) + ] + connections = [StageConnection(before, line[0], {}, entries[0])] + for position, stage in enumerate(line): + consumer = line[position + 1] if position + 1 < len(line) else after + following = entries[position + 1] if consumer is not None else {} + connections.append( + StageConnection( + stage, + consumer, + { + v: _exit_edge(stage, flows[position][v], following.get(v)) + for v in variants + }, + following, + ) ) - entries[variant] = edge - if ( - before is not None - and before.kind is StageKind.ATTENTION - and before.reduction is ProducerReduction.ALWAYS_PARTIAL - ): - exits[variant] = edge - return StageConnection(producer, consumer, exits, entries) + return connections -def _fork_input(prepared, consumer): - """Place an already-read input onto a branch's computation rows. - - The branch adapter performs the move and forks the stream. It does not - execute the branch's input norm again. - """ - entries = {} +def _fork_input(prepared): + """What reaches a branch's first stage: an already-read input, placed onto + the branch's computation rows by the branch adapter, which forks the + stream without running the branch's input norm again.""" + arrivals = {} for variant in _active_variants(): if variant is not BatchVariant.ORDINARY: raise NotImplementedError( "prepared branch transport requires ordinary token rows" ) source = prepared.entries[variant] - declaration, during, _ = _resolve_stage(consumer, variant) - entries[variant] = EdgeContract( + arrivals[variant] = _Arrival( OutputContract(source.need.layout, update=None), - declaration.input, source.residual_to, - during, - arriving_plain_add=True, + True, + False, ) - return StageConnection(prepared.consumer, consumer, {}, entries) - - -def _incoming(stage, previous_incoming=None): - """The boundary into ``stage``. ``previous_incoming`` is the boundary its - producer was bound with, when that producer ended the previous append and - its own history is no longer on ``stage``.""" - if stage.prepared_from is not None: - return _fork_input(_incoming(stage.prepared_from), stage) - previous = stage.previous - # A normal attention may retain a finer residual than its output rows. - # FFN and mixer exits declare their returned residual placement directly. - source = None - if ( - previous is not None - and previous.kind is StageKind.ATTENTION - and previous.reduction is ProducerReduction.ALWAYS_PARTIAL - ): - source = ( - previous_incoming if previous_incoming is not None else _incoming(previous) - ) - return _connect( - previous, - stage, - residual_from=source, - ) + return arrivals class _Append: @@ -859,58 +894,6 @@ def _note_origin(error: Exception, origin: str) -> None: add_note(f"while binding the stages appended at {origin}") -class _PendingStage: - """The last stage of a linear append. Its consumer is the next append's - first stage, or the stack exit, so it binds only once that is known.""" - - __slots__ = ( - "boundary", - "declaration", - "norm", - "options", - "predecessor", - "predecessor_incoming", - "predecessor_boundary", - "origin", - ) - - def __init__( - self, - boundary, - declaration, - norm, - options, - predecessor, - predecessor_incoming, - predecessor_boundary, - origin, - ): - self.boundary = boundary - self.declaration = declaration - self.norm = norm - self.options = options - # The stage before it within the same append, or None when it is the - # append's only stage and takes its input from the stack. - self.predecessor = predecessor - self.predecessor_incoming = predecessor_incoming - self.predecessor_boundary = predecessor_boundary - self.origin = origin - - -class _Chain: - """The stage a following append extends, and the one stage still waiting - for its consumer, while the stack binds its appends in order.""" - - __slots__ = ("previous", "pending", "previous_incoming") - - def __init__(self, previous): - self.previous = _detached(_handed_off(previous)) - self.pending = None - # The boundary the stage ``previous`` stands for was bound with, once - # it is bound on this rank: ``previous`` itself drops that history. - self.previous_incoming = None - - def _handed_off(declaration): """A stage whose output crosses to another pipeline rank: an FFN leaves it on the attention's rows, which is what the handoff carries and the next @@ -956,51 +939,94 @@ def _return_before_trailing_attention(line): def _detached(declaration): - """The declaration a following append extends, without its own history. - - A stage's incoming edge reads its producer's own incoming edge only when - the producer is an attention that always leaves its sum; the copy has no - history, so that lookback stops at the producer. - """ + """The declaration as the stage after it records it: without its own + history, so a bound stage holds its producer's facts and not the chain of + declarations before it.""" if declaration is None: return None return replace(declaration, previous=None, prepared_from=None) def _bind_stack(appends, *, previous, following, final_read=None): - """Bind every appended stage, in order, and fill in the boundaries each - append returned.""" + """Bind every appended stage and fill in the boundaries each append + returned. + + The stages of the appends that extend the stack form one line, chained in + append order, and every connection between two of them is built once: + each stage's outgoing connection is the next one's incoming. A branch + then starts from the incoming connection of the stage it branches from. + """ line = [a for a in appends if a.prepared_from is None] if following is not None: # The last stage hands off to the next rank. line[-1].declarations[-1] = _handed_off(line[-1].declarations[-1]) _return_before_trailing_attention(line) - chain = _Chain(previous) - # A returned declaration's boundary as bound, for the branches that read it. + if following is None: + # Without a stage on a later rank, the last one ends the model's stack, + # read by the final read. + line[-1].declarations[-1] = _ended( + replace(line[-1].declarations[-1], terminal=True), final_read + ) + stages = [ + (append, index) for append in line for index in range(len(append.bindings)) + ] + bound = {id(append): [None] * len(append.bindings) for append in appends} + # A returned declaration's stage as bound, for the branches that start from it. sources = {} - bound = [] - for append in appends: - prepared_from = append.prepared_from - if prepared_from is not None: - source = sources.get(id(prepared_from)) - if source is None: - raise ValueError( - "prepared_from must be the declaration of a stage appended " - f"earlier to the same layer stack (at {append.origin})" - ) - prepared_from = source.declaration - boundaries = _extend(chain, append, prepared_from) - for returned, boundary in zip(append.boundaries, boundaries): - sources[id(returned.declaration)] = boundary - bound.append(boundaries) - # The last stage's consumer is the next rank's first stage, if any; - # without one it ends the model's layer stack, read by the final read. - _bind_pending( - chain, - consumer=following, - terminal=following is None, + producer = _detached(_handed_off(previous)) + chained = [] + for append, index in stages: + chained.append( + replace(append.declarations[index], previous=producer, prepared_from=None) + ) + producer = _detached(chained[-1]) + connections = _connect_line( + chained, + [append.origin for append, _ in stages], + before=chained[0].previous, + after=following, + ) + _bind_line( + stages, + chained, + connections, + bound, + sources, final_read=final_read if following is None else None, ) + for append in appends: + if append.prepared_from is None: + continue + source = sources.get(id(append.prepared_from)) + if source is None: + raise ValueError( + "prepared_from must be the declaration of a stage appended " + f"earlier to the same layer stack (at {append.origin})" + ) + source_declaration, source_incoming = source + branch = [(append, index) for index in range(len(append.bindings))] + chained = [] + for index, declaration in enumerate(append.declarations): + chained.append( + replace( + declaration, + previous=chained[-1] if chained else None, + prepared_from=None if chained else source_declaration, + ) + ) + try: + arrivals = _fork_input(source_incoming) + except Exception as error: + _note_origin(error, append.origin) + raise + connections = _connect_line( + chained, + [append.origin] * len(chained), + before=source_declaration, + arrivals=arrivals, + ) + _bind_line(branch, chained, connections, bound, sources) + bound = [tuple(bound[id(append)]) for append in appends] _check_declared_gathers(appends, bound, remote_producer=previous is not None) for append, boundaries in zip(appends, bound): for returned, boundary in zip(append.boundaries, boundaries): @@ -1008,6 +1034,72 @@ def _bind_stack(appends, *, previous, following, final_read=None): returned.declaration = boundary.declaration +def _bind_line(stages, chained, connections, bound, sources, *, final_read=None): + """Bind a chain of stages from its connections, ``connections[i]`` into + ``chained[i]`` and ``connections[i + 1]`` out of it. An attention that + always leaves its sum binds after the FFN that follows it in the same + append, so the attention's entry keeps the residual that FFN's capture + needs.""" + last = len(chained) - 1 + boundaries = [None] * len(chained) + + def bind(position): + append, index = stages[position] + norm, options = append.bindings[index] + consumer = None + if position < last and stages[position + 1][0] is append: + consumer = chained[position + 1] + if _carries_capture(chained[position], consumer): + bind(position + 1) + try: + boundary = _bind_stage( + chained[position], + norm, + connections[position], + connections[position + 1], + final_read=final_read if position == last else None, + capture_preserves_residual=( + _preserves_residual(boundaries[position + 1]) + if consumer is not None + and _carries_capture(chained[position], consumer) + else None + ), + **options, + ) + except Exception as error: + _note_origin(error, append.origin) + raise + boundaries[position] = boundary + bound[id(append)][index] = boundary + sources[id(append.boundaries[index].declaration)] = ( + boundary.declaration, + connections[position], + ) + + for position in range(len(chained)): + if boundaries[position] is None: + bind(position) + + +def _carries_capture(producer, consumer): + """Whether an attention that always leaves its sum keeps the residual its + FFN's entry needs for capture. Only between stages of one append.""" + return ( + producer.kind is StageKind.ATTENTION + and producer.reduction is ProducerReduction.ALWAYS_PARTIAL + and consumer.kind is StageKind.FFN + ) + + +def _preserves_residual(consumer): + """Each variant's check that the FFN's entry leaves the residual in place.""" + return { + variant: path.entry.preserves_residual + for variant, path in consumer.plan.paths.items() + if path.entry.preserves_residual is not None + } + + def _check_declared_gathers(appends, bound, *, remote_producer): """A stage that declares its gather over attention TP must have one: at its own entry, or at its producer's exit. A producer on another pipeline @@ -1048,63 +1140,6 @@ def _ended(declaration, final_read): return declaration -def _bind_pending(chain: _Chain, *, consumer, terminal, final_read=None): - pending = chain.pending - if pending is None: - return - chain.pending = None - try: - declaration = replace(pending.declaration, terminal=terminal) - if terminal: - declaration = _ended(declaration, final_read) - if pending.predecessor is None: - incoming = _incoming(declaration, chain.previous_incoming) - else: - incoming = _connect( - pending.predecessor, - declaration, - residual_from=pending.predecessor_incoming, - ) - outgoing = _connect(declaration, consumer, residual_from=incoming) - bound = _bind_stage( - declaration, - pending.norm, - incoming, - outgoing, - final_read=final_read, - **pending.options, - ) - except Exception as error: - _note_origin(error, pending.origin) - raise - pending.boundary.plan = bound.plan - pending.boundary.declaration = declaration - chain.previous_incoming = incoming - if pending.predecessor_boundary is not None: - _carry_capture(pending.predecessor_boundary, pending.boundary) - - -def _carry_capture(producer, consumer): - """Let an attention that always leaves its sum preserve the residual its - FFN's entry needs for capture. Only between stages of one append.""" - if not ( - producer.kind is StageKind.ATTENTION - and producer.declaration.reduction is ProducerReduction.ALWAYS_PARTIAL - and consumer.kind is StageKind.FFN - ): - return - for variant, steps in producer.plan.paths.items(): - next_steps = consumer.plan.paths[variant] - predicate = next_steps.entry.preserves_residual - if predicate is not None: - producer.plan.paths[variant] = msgspec.structs.replace( - steps, - entry=msgspec.structs.replace( - steps.entry, capture_preserves_residual=predicate - ), - ) - - _STAGE_OPTIONS = { StageKind.ATTENTION: frozenset({"qkv_latent_func", "fusions"}), StageKind.FFN: frozenset({"fusions"}), @@ -1172,64 +1207,3 @@ def append_stages(*stages, prepared_from=None): _Append(declarations, bindings, prepared_from, boundaries, _origin(stack)) ) return boundaries - - -def _extend(chain, append, prepared_from): - """Chain one append onto the stack and bind what can be bound.""" - declarations, bindings = append.declarations, append.bindings - branch = prepared_from is not None - if not branch: - # This append's first stage is the consumer the pending stage waited for. - _bind_pending(chain, consumer=declarations[0], terminal=False) - try: - chained = [] - for index, declaration in enumerate(declarations): - chained.append( - replace( - declaration, - previous=( - chained[-1] if index else (None if branch else chain.previous) - ), - prepared_from=prepared_from if index == 0 else None, - ) - ) - boundaries = [] - last = len(chained) - 1 - # incoming feeds the stage being bound; previous_incoming fed the one - # before. - previous_incoming, incoming = ( - None, - _incoming(chained[0], None if branch else chain.previous_incoming), - ) - for index, (declaration, (norm, options)) in enumerate(zip(chained, bindings)): - if index == last and not branch: - boundaries.append(StageBoundary(None, declaration=declaration)) - chain.pending = _PendingStage( - boundaries[-1], - declaration, - norm, - options, - predecessor=chained[index - 1] if index else None, - predecessor_incoming=previous_incoming, - predecessor_boundary=boundaries[-2] if index else None, - origin=append.origin, - ) - break - following = chained[index + 1] if index < last else None - outgoing = _connect(declaration, following, residual_from=incoming) - boundaries.append( - _bind_stage(declaration, norm, incoming, outgoing, **options) - ) - previous_incoming, incoming = incoming, outgoing - except Exception as error: - _note_origin(error, append.origin) - raise - if branch: - for producer, consumer in zip(boundaries, boundaries[1:]): - _carry_capture(producer, consumer) - else: - # The pair that ends on the pending stage is carried when it binds. - for producer, consumer in zip(boundaries[:-2], boundaries[1:-1]): - _carry_capture(producer, consumer) - chain.previous = _detached(chained[-1]) - return tuple(boundaries) diff --git a/test/registered/unit/layer_boundary/test_independent_stage_construction.py b/test/registered/unit/layer_boundary/test_independent_stage_construction.py index 63504a7e5e..975c530441 100644 --- a/test/registered/unit/layer_boundary/test_independent_stage_construction.py +++ b/test/registered/unit/layer_boundary/test_independent_stage_construction.py @@ -242,7 +242,10 @@ class TestIndependentStageConstruction(CustomTestCase): following.kind is StageKind.ATTENTION, ) incoming = consumer.plan.edges[variant].incoming.produced - self.assertEqual(incoming.group, output.group) + self.assertEqual( + incoming.group, + output.group if output.always_partial else None, + ) # The kinds of stage a stack is cut between. diff --git a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py index f4b9f6a038..a6c1c8b025 100644 --- a/test/registered/unit/layer_boundary/test_nemotron_h_stages.py +++ b/test/registered/unit/layer_boundary/test_nemotron_h_stages.py @@ -28,7 +28,7 @@ register_cpu_ci(est_time=3, suite="base-a-test-cpu") def layer_stage(pattern, index): from sglang.srt.layers.layer_boundary.construction import BatchVariant - from sglang.srt.layers.layer_boundary.factories import _connect, _incoming + from sglang.srt.layers.layer_boundary.factories import _connect_line previous = utils._declaration(pattern, index - 1) if index else None declaration = replace( @@ -41,8 +41,9 @@ def layer_stage(pattern, index): if index + 1 < len(pattern) else None ) - incoming = _incoming(declaration) - outgoing = _connect(declaration, following, residual_from=incoming) + incoming, outgoing = _connect_line( + [declaration], [pattern], before=previous, after=following + ) return SimpleNamespace( kind=declaration.kind, edges=( @@ -118,17 +119,14 @@ class TestStageEdges(CustomTestCase): self.assertEqual(out_of.residual_to, rows) self.assertEqual(into.residual, rows) self.assertEqual(into.produced.layout, rows) + # The next entry owes the sum the exit always leaves. produced = out_of.produced - may_leave = produced.always_partial or produced.may_defer_to_next - self.assertEqual( - into.produced.group, produced.group if may_leave else None - ) self.assertEqual( into.produced.always_partial, produced.always_partial ) self.assertEqual( - into.produced.may_defer_to_next, - produced.may_defer_to_next, + into.produced.group, + produced.group if produced.always_partial else None, ) self.assertEqual( layers[0].edges[0].produced, OutputContract(rows, update=None) @@ -139,7 +137,7 @@ class TestStageEdges(CustomTestCase): self.assertFalse(last.always_partial or last.may_defer_to_next) def test_what_each_kind_of_boundary_carries(self): - # (pattern, boundary after layer 0): group, always_partial, may_defer_to_next + # (pattern, layer 0's output): group, always_partial, may_defer_to_next cases = { "M-": (SumGroup.ATTN_TP, True, False), "*E": (SumGroup.ATTN_TP, True, False), @@ -151,9 +149,9 @@ class TestStageEdges(CustomTestCase): } for pattern, expected in cases.items(): with self.subTest(pattern=pattern): - into = stages(pattern, tp=2)[1].edges[0].produced + out = stages(pattern, tp=2)[0].edges[1].produced self.assertEqual( - (into.group, into.always_partial, into.may_defer_to_next), + (out.group, out.always_partial, out.may_defer_to_next), expected, ) # Without attention TP a mixer's output is complete. @@ -163,9 +161,9 @@ class TestStageEdges(CustomTestCase): # backend dispatches only the MoE, so an MLP still sums over TP. into = stages("EM", tp=2, a2a=True)[1].edges[0].produced self.assertEqual(into, OutputContract(into.layout, update=None)) - into = stages("-M", tp=2, a2a=True)[1].edges[0].produced + out = stages("-M", tp=2, a2a=True)[0].edges[1].produced self.assertEqual( - (into.group, into.always_partial, into.may_defer_to_next), + (out.group, out.always_partial, out.may_defer_to_next), (SumGroup.TP, False, True), )