diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 21559d1ba5..32f3003729 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -17,9 +17,10 @@ from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.communicator import ( CommunicateContext, Layout, - reduce_output, tbo_split_moves, ) +from sglang.srt.layers.communicator.residual.access import finish_layer_stack +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, @@ -964,12 +965,25 @@ def _model_forward_tbo( operations_strategy: OperationsStrategy, layer_input_rows: Layout, ): - inputs["hidden_states"] = reduce_output(inputs["hidden_states"]) + parent_batch = inputs["forward_batch"] + stream = inputs["residual"] + pending = stream.pending if isinstance(stream, ResidualStream) else None + inputs["hidden_states"], inputs["residual"] = finish_layer_stack( + inputs["hidden_states"], inputs["residual"], inputs["forward_batch"] + ) inputs_arr = _model_forward_tbo_split_inputs( **inputs, layer_input_rows=layer_input_rows ) original_hidden_states_len = inputs["hidden_states"].shape[0] - del inputs + if isinstance(stream, ResidualStream): + for part in inputs_arr: + part["hidden_states"], part["residual"] = ResidualStream.arrive( + part["hidden_states"], + part["residual"], + pending.update if pending is not None else None, + ) + part["forward_batch"].residual_stream = part["residual"] + del inputs, stream, pending context = ( empty_context() @@ -986,7 +1000,13 @@ def _model_forward_tbo( delta_stages=[0, operations_strategy.tbo_delta_stages], ) - return _model_forward_tbo_merge_outputs(*outputs_arr, original_hidden_states_len) + hidden_states, residual = _model_forward_tbo_merge_outputs( + *outputs_arr, original_hidden_states_len + ) + parent_batch.residual_stream = ( + residual if isinstance(residual, ResidualStream) else None + ) + return hidden_states, residual def _model_forward_non_tbo(inputs, operations_strategy: OperationsStrategy): @@ -1100,6 +1120,21 @@ def _model_forward_filter_inputs( def _model_forward_tbo_merge_outputs(output_a, output_b, original_len): + stream_a, stream_b = output_a["residual"], output_b["residual"] + has_stream = isinstance(stream_a, ResidualStream) + assert has_stream == isinstance(stream_b, ResidualStream) + update = None + if has_stream: + pending_a, pending_b = stream_a.pending, stream_b.pending + assert (pending_a is None) == (pending_b is None) + if pending_a is not None: + assert pending_a.update is pending_b.update + update = pending_a.update + for output in (output_a, output_b): + output["hidden_states"], output["residual"] = finish_layer_stack( + output["hidden_states"], output["residual"], output["forward_batch"] + ) + def _handle_key(name): value_a = output_a[name] value_b = output_b[name] @@ -1117,7 +1152,12 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len): res[slice(s1, t1)] = value_b[: t1 - s1] return res - return _handle_key("hidden_states"), _handle_key("residual") + hidden, residual = _handle_key("hidden_states"), _handle_key("residual") + return ( + ResidualStream.arrive(hidden, residual, update) + if has_stream + else (hidden, residual) + ) # -------------------------------- Utilities and wrappers --------------------------------------- diff --git a/python/sglang/srt/layers/communicator/boundary.py b/python/sglang/srt/layers/communicator/boundary.py index 3d75f47663..3fc3710ff9 100644 --- a/python/sglang/srt/layers/communicator/boundary.py +++ b/python/sglang/srt/layers/communicator/boundary.py @@ -35,6 +35,7 @@ from sglang.srt.layers.communicator.ops import ( CommunicateSimpleFn, CommunicateSummableTensorPairFn, _consumer_step, + _dispatch_consumer, _hand_qkv_hook_its_input, _mlp_input_dp_partial, _mlp_input_dp_replicate, @@ -96,8 +97,9 @@ class StageOutput(msgspec.Struct, frozen=True): # Whether it leaves the sum to the attention-DP reduce_scatterv whenever that # combine applies, as a MoE block does without any flag. leaves_for_reduce_scatterv: bool = False - # How the producer's output is written into the residual. - update: StageUpdate = ADD + # How the producer's output is written into the residual. An arrival + # description has no producer object; its edge declares capabilities only. + update: Optional[StageUpdate] = ADD class StageDecl(msgspec.Struct, frozen=True): @@ -140,6 +142,8 @@ class EdgeDecl(msgspec.Struct, frozen=True): # Whether the residual is added into one rank's share of the produced sum # before that sum completes, instead of after it. residual_joins_sum: bool = False + # Capabilities allowed to arrive from another layer, not its update object. + update_capabilities: Tuple[bool, ...] = () class DecoderLayerEdges(msgspec.Struct, frozen=True): @@ -159,13 +163,13 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: construction (``input_owes``); a sum the previous layer leaves for a batch comes with the value.""" owes = sides.input_owes is not None - # The previous layer's FFN output, written in as this layer's FFN writes its - # own. + # Cross-layer arrival describes rows and owed work. The actual update + # arrives in the stream; this model family supplies only its capability. arrived = StageOutput( sides.input_rows, group=sides.input_owes, always_leaves=owes, - update=sides.ffn.output.update, + update=None, ) # Completing what the input owes leaves it and the residual on each # attention-TP rank's slice. @@ -180,6 +184,7 @@ def decoder_layer_edges(sides: DecoderLayerSides) -> DecoderLayerEdges: need=sides.attention.input, residual=sides.input_rows, residual_to=attention_rows, + update_capabilities=(sides.ffn.output.update.adds_plainly,), ), into_ffn=EdgeDecl( produced=sides.attention.output, @@ -227,22 +232,28 @@ def stage_edges( first, and what it may leave of its sum comes with the value; otherwise the value is complete. The residual follows the input onto a finer slice and stays where it is when the input is gathered.""" - update = previous.update if previous is not None else ADD + adds_plainly = previous.update.adds_plainly if previous is not None else True arrived = ( StageOutput( rows, group=previous.group, always_leaves=previous.always_leaves, leaves_for_next_layer=previous.leaves_for_next_layer, - update=update, + update=None, ) if previous is not None and (previous.always_leaves or previous.leaves_for_next_layer) - else StageOutput(rows, update=update) + else StageOutput(rows, update=None) ) during = stage.input.layout if rows.sharded <= stage.input.layout.sharded else rows return ( - EdgeDecl(produced=arrived, need=stage.input, residual=rows, residual_to=during), + EdgeDecl( + produced=arrived, + need=stage.input, + residual=rows, + residual_to=during, + update_capabilities=(adds_plainly,), + ), EdgeDecl( produced=stage.output, need=StageInput(rows), @@ -622,6 +633,56 @@ def make_boundary( force_layernorm_before_gather: bool = False, cp_moves: Optional[CpMoves] = None, enters_stack: bool = False, +) -> Boundary: + """Bind only the update capabilities that can reach this edge. + + Execution receives the producer's actual update from the residual stream. + A single capability needs no dispatcher; multiple capabilities select one + of the already constructed paths at the consumer. + """ + capabilities = edge.update_capabilities + if not capabilities: + if edge.produced.update is None: + raise ValueError("an arrival must declare its update capabilities") + capabilities = (edge.produced.update.adds_plainly,) + paths = { + capability: _bind_consumer( + edge, + adds_plainly=capability, + fusions=fusions, + carried_fusions=carried_fusions, + force_layernorm_before_gather=force_layernorm_before_gather, + cp_moves=cp_moves, + enters_stack=enters_stack, + ) + for capability in capabilities + } + first = next(iter(paths.values())) + if len(paths) == 1: + return first + if any(path.input_move != first.input_move for path in paths.values()): + raise NotImplementedError("update capabilities require different input moves") + return msgspec.structs.replace( + first, + prepare=partial( + _dispatch_consumer, + paths={capability: path.prepare for capability, path in paths.items()}, + ), + fused=tuple( + f for f in fusions if any(f in path.fused for path in paths.values()) + ), + ) + + +def _bind_consumer( + edge: EdgeDecl, + *, + adds_plainly: bool, + fusions: Tuple["FusedMlpInput", ...] = (), + carried_fusions: Tuple[Callable, ...] = (), + force_layernorm_before_gather: bool = False, + cp_moves: Optional[CpMoves] = None, + enters_stack: bool = False, ) -> Boundary: """The consumer's half of ``edge``, chosen from the edge's declarations and the producer's update: what a value carries for a batch (a sum or a handoff @@ -633,28 +694,45 @@ def make_boundary( over attention CP; ``enters_stack`` for the edge into the layer stack's first stage. The steps read only this edge's declarations, never what the producer chose for a batch.""" - update = edge.produced.update # A fused kernel runs the add and the norm itself. - plain = update.adds_plainly and edge.need.read.norms_plainly + plain = adds_plainly and edge.need.read.norms_plainly step, fused, input_move = _select_input_steps( edge.produced, residual=edge.residual, residual_to=edge.residual_to, need=edge.need, - update=update, + adds_plainly=adds_plainly, fusions=fusions if plain else (), force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=edge.residual_joins_sum, cp_moves=cp_moves, enters_stack=enters_stack, ) + # A written stream entering the first physical layer must not run enter + # again (e.g. MHC expansion). Both alternatives are bound at construction. + written_step = None + if enters_stack: + written_step, _, _ = _select_input_steps( + edge.produced, + residual=edge.residual, + residual_to=edge.residual_to, + need=edge.need, + adds_plainly=adds_plainly, + fusions=fusions if plain else (), + force_layernorm_before_gather=force_layernorm_before_gather, + residual_joins_sum=edge.residual_joins_sum, + cp_moves=cp_moves, + enters_stack=False, + ) return Boundary( edge, prepare=partial( _consumer_step, step=step, + adds_plainly=adds_plainly, carried_fusions=carried_fusions if plain else (), owes_by_construction=edge.produced.always_leaves, + written_step=written_step, ), input_move=input_move, fused=fused, @@ -668,6 +746,16 @@ def make_output_boundary( rows the layer hands on (``edge.need``), whose consumer runs in the next layer: the postprocess that moves the output there. ``cp_moves`` for an edge that returns across attention CP.""" + update = edge.produced.update + if not getattr(update, "at_producer", False): + if not getattr(update, "can_defer_across_layers", False): + raise NotImplementedError( + "a deferred update must guarantee its lifetime across layers" + ) + if not update.adds_plainly and get_parallel().pp_size > 1: + raise NotImplementedError( + "pipeline boundaries require a plain add or a producer-written residual" + ) if edge.need.layout != edge.residual_to: raise NotImplementedError(f"{edge=}") returns_over_dp, output_move, completes_sum = _select_ffn_output_move( @@ -740,7 +828,7 @@ def _select_input_steps( residual: Layout, residual_to: Layout, need: StageInput, - update: StageUpdate, + adds_plainly: bool, fusions: Tuple[FusedMlpInput, ...], force_layernorm_before_gather: bool, residual_joins_sum: bool, @@ -781,7 +869,6 @@ def _select_input_steps( _mlp_input_scatter if owes is SumGroup.ATTN_TP else _mlp_input_slice, scatters_residual=residual != residual_to, read=read, - update=update, ), (), None, @@ -793,7 +880,7 @@ def _select_input_steps( if gathered or owes is not SumGroup.ATTN_TP: raise NotImplementedError(f"{produced=} {residual=} {need=}") return ( - partial(_mlp_input_on_residual_shard, read=read, update=update), + partial(_mlp_input_on_residual_shard, read=read), (), None, ) @@ -812,7 +899,6 @@ def _select_input_steps( layer_input=tp_reduce_scatter, enters_stack=enters_stack, read=read, - update=update, ), (), None, @@ -826,7 +912,7 @@ def _select_input_steps( residual=residual, residual_to=residual_to, need=StageInput(produced.layout, read=read), - update=update, + adds_plainly=adds_plainly, fusions=fusions, force_layernorm_before_gather=force_layernorm_before_gather, residual_joins_sum=residual_joins_sum, @@ -849,7 +935,6 @@ def _select_input_steps( layer_input=None, enters_stack=enters_stack, read=read, - update=update, ), (), CommunicateSimpleFn._scattered_to_tp_attn_full, @@ -878,7 +963,6 @@ def _select_input_steps( layer_input=None, enters_stack=enters_stack, read=read, - update=update, ), (), None, @@ -886,7 +970,7 @@ def _select_input_steps( if gathers_residual and residual_joins_sum: # Each rank adds its slice of the residual into its share of the # sum, so the all-reduce also brings the residual back to every row. - if owes is not SumGroup.ATTN_TP or not update.adds_plainly: + if owes is not SumGroup.ATTN_TP or not adds_plainly: raise NotImplementedError(f"{produced=} {residual=} {need=}") return partial(_mlp_input_residual_into_sum, read=read), (), None # A sum over TP completes only on rows every TP rank holds: the TP group @@ -903,7 +987,6 @@ def _select_input_steps( fusions=tuple(f.run for f in fused), group=owes, read=read, - update=update, ), fused, None, @@ -920,7 +1003,7 @@ def _select_input_steps( if ( owes_attention_tp and not force_layernorm_before_gather - and update.adds_plainly + and adds_plainly and read.norms_plainly ): return ( @@ -940,7 +1023,6 @@ def _select_input_steps( reduces_attention_tp=owes_attention_tp, places_cp_shards=places_cp_shards, read=read, - update=update, ), (), None, diff --git a/python/sglang/srt/layers/communicator/layer.py b/python/sglang/srt/layers/communicator/layer.py index 2620d20f1f..5101bea133 100644 --- a/python/sglang/srt/layers/communicator/layer.py +++ b/python/sglang/srt/layers/communicator/layer.py @@ -72,6 +72,7 @@ from sglang.srt.layers.communicator.output import ( reduce_output, ) from sglang.srt.layers.communicator.residual import LayerResidual +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.communicator.residual.access import ( add_to_output, buffer, @@ -79,6 +80,7 @@ from sglang.srt.layers.communicator.residual.access import ( fold, from_pp, snapshot, + written, ) from sglang.srt.layers.communicator.residual.add_norm import ( PLAIN_RESIDUAL, @@ -87,6 +89,7 @@ from sglang.srt.layers.communicator.residual.add_norm import ( apply_flashinfer_allreduce_fusion, ) from sglang.srt.layers.communicator.residual.mhc import MHCState +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, is_enable_moe_cp_allgather, @@ -195,6 +198,15 @@ class StageCommunicator: ``quant_format``).""" entry = self.entry(forward_batch, steps) context = self._layer._context + stream = residual_batch.current(forward_batch) + if residual is not stream: + raise RuntimeError("residual alias belongs to a different invocation") + if stream is not None: + if stream.pending is not None: + call["update"] = stream.pending.update + if stream.pending is None and stream.residual is not None: + call["written"] = True + hidden_states, residual = stream.input(hidden_states) hidden_states, residual = entry.prepare( hidden_states, residual, forward_batch, self.norm, context, **call ) @@ -208,7 +220,8 @@ class StageCommunicator: hidden_states = entry.handoff( hidden_states, forward_batch, self._layer.qkv_latent_func ) - return hidden_states, residual + stream.write(residual) + return hidden_states, stream class LayerFusions(Protocol): @@ -231,6 +244,7 @@ class LayerCommunicator: add_to_output = staticmethod(add_to_output) buffer = staticmethod(buffer) fold = staticmethod(fold) + written = staticmethod(written) finish_layer_stack = staticmethod(finish_layer_stack) # Communicators built without __init__ (e.g. test doubles) publish no LoRA @@ -602,15 +616,24 @@ class LayerCommunicator: storage = buffer(hidden_states) if storage is not None and storage.shape[0] == 0: return hidden_states, None - hidden_states = reduce_output(hidden_states) + hidden_states = ( + residual.complete(hidden_states) + if isinstance(residual, ResidualStream) + else reduce_output(hidden_states) + ) return hidden_states, self.snapshot(hidden_states, residual, at_input=at_input) - def from_pp(self, tensors, *, allow_missing_residual: bool = False): - return from_pp( + def from_pp(self, tensors, forward_batch, *, allow_missing_residual: bool = False): + hidden_states, residual = from_pp( tensors, residual_in_hidden=self._residual.ffn_update.at_producer, allow_missing_residual=allow_missing_residual, ) + hidden_states, stream = ResidualStream.arrive( + hidden_states, residual, self._residual.ffn_update + ) + forward_batch.residual_stream = stream + return hidden_states, stream def prepare_attn_and_capture_last_layer_outputs( self, @@ -629,23 +652,26 @@ class LayerCommunicator: post_residual_addition=post_residual_addition, ) if captured_last_layer_outputs is not None: + residual_value = residual.residual move = self.attn.entry(forward_batch).input_move gathered_last_layer_output = ( - residual + residual_value if move is None else move( - hidden_states=residual, + hidden_states=residual_value, forward_batch=forward_batch, context=self._context, ) ) if ( - gathered_last_layer_output is residual + gathered_last_layer_output is residual_value # An accumulator that copies on append already holds a snapshot. and not getattr(captured_last_layer_outputs, "copies_on_append", False) - and not self._post_attn_residual_is_read_only(residual, forward_batch) + and not self._post_attn_residual_is_read_only( + residual_value, forward_batch + ) ): - gathered_last_layer_output = residual.clone() + gathered_last_layer_output = residual_value.clone() captured_last_layer_outputs.append(gathered_last_layer_output) return hidden_states, residual @@ -836,7 +862,18 @@ class LayerCommunicator: if cache is not None: self._context.cache = cache - return self.ffn.prepare(hidden_states, residual, forward_batch, steps) + if isinstance(residual, ResidualStream) and self.stage_edges is None: + # The paired boundary records the attention contribution once. + hidden_states = residual.leave( + hidden_states, self._residual.attention_update + ) + return self.ffn.prepare( + hidden_states, + residual, + forward_batch, + steps, + update=self._residual.attention_update, + ) def maybe_prefetch_next_full_attention_kv( self, @@ -851,12 +888,28 @@ class LayerCommunicator: residual: torch.Tensor, forward_batch: ForwardBatch, ): - return self._complete_ffn_output_now( + stream = residual if isinstance(residual, ResidualStream) else None + hidden_states, residual = self._complete_ffn_output_now( hidden_states, - residual, + stream.residual if stream is not None else residual, forward_batch=forward_batch, dp_step=self._postprocess_dp_step(forward_batch), ) + return self._leave_ffn_output( + hidden_states, + residual, + stream, + self._batch_steps(forward_batch).ffn_output.update, + ) + + @staticmethod + def _leave_ffn_output(hidden_states, residual, stream, update): + if stream is None: + return hidden_states, residual + stream.residual = residual + if update.at_producer: + return stream.write(hidden_states), stream + return stream.leave(hidden_states, update), stream def _local_token_move_can_go_to_next_layer( self, forward_batch: ForwardBatch @@ -1037,10 +1090,15 @@ class LayerCommunicator: to the rows this FFN needs and its residual's rows.""" rows, residual_rows, _ = source._branch_rows(forward_batch) to, residual_to, _ = self._branch_rows(forward_batch) - return ( - move_rows(hidden_states, rows, to, forward_batch), - move_rows(residual, residual_rows, residual_to, forward_batch), - ) + stream = residual if isinstance(residual, ResidualStream) else None + if stream is not None: + if stream.pending is not None: + raise RuntimeError("a branch must start from a prepared stage input") + residual = stream.residual + hidden_states = move_rows(hidden_states, rows, to, forward_batch) + residual = move_rows(residual, residual_rows, residual_to, forward_batch) + forward_batch.residual_stream = ResidualStream(residual) + return hidden_states, forward_batch.residual_stream def branch_output( self, hidden_states: torch.Tensor, forward_batch: ForwardBatch @@ -1064,9 +1122,19 @@ class LayerCommunicator: this layer hands on.""" _, _, rows = source._branch_rows(forward_batch) _, _, to = self._branch_rows(forward_batch) + stream = residual if isinstance(residual, ResidualStream) else None + update = None + if stream is not None: + stream.check(hidden_states) + update = stream.pending.update + hidden_states, residual = stream.finish(hidden_states) hidden_states = move_rows(hidden_states, rows, to, forward_batch) residual = move_rows(residual, rows, to, forward_batch) - return contribution + hidden_states, residual + output = contribution + hidden_states + if stream is not None: + stream.write(residual) + return stream.leave(output, update), stream + return output, residual def _select_ffn_exit_fusions( self, @@ -1203,10 +1271,11 @@ class MixerExit: its input), and when it may leave it and the fused kernel takes it into the next input norm.""" - __slots__ = ("skips_reduction", "_hands_on", "_scope") + __slots__ = ("skips_reduction", "_hands_on", "_scope", "_update") def __init__(self, communicator: LayerCommunicator, forward_batch: ForwardBatch): produced = communicator._batch_steps(forward_batch).ffn_output + self._update = produced.update self._hands_on = ( produced.leaves_for_next_layer and communicator.should_fuse_mlp_allreduce_with_next_layer(forward_batch) @@ -1221,13 +1290,15 @@ class MixerExit: def __exit__(self, *exc_info): return self._scope.__exit__(*exc_info) - def finish( - self, hidden_states: torch.Tensor - ) -> Union[torch.Tensor, UnreducedOutput]: + def finish(self, hidden_states: torch.Tensor, residual=None): """The mixer's output: its partial sum to the FFN stage after it, as a value that owes the sum to a mixer after it, or complete.""" if self._hands_on: - return UnreducedOutput(hidden_states, group=get_parallel().tp_group) + hidden_states = UnreducedOutput( + hidden_states, group=get_parallel().tp_group + ) + if isinstance(residual, ResidualStream): + return residual.leave(hidden_states, self._update), residual return hidden_states @@ -1243,6 +1314,7 @@ class FfnExit: "fuse_mlp_allreduce", "mlp_reduce_scatter", "_complete", + "_update", "_scope", ) @@ -1254,6 +1326,7 @@ class FfnExit: self.fuse_mlp_allreduce = completion.fuse_mlp_allreduce self.mlp_reduce_scatter = completion.mlp_reduce_scatter self._complete = completion.complete + self._update = communicator._batch_steps(forward_batch).ffn_output.update self._scope = get_forward().scoped( fuse_mlp_allreduce=self.fuse_mlp_allreduce, mlp_reduce_scatter=self.mlp_reduce_scatter, @@ -1271,11 +1344,17 @@ class FfnExit: self, hidden_states: torch.Tensor, residual: torch.Tensor ) -> Tuple[Union[torch.Tensor, UnreducedOutput], torch.Tensor]: """Leave the reduction to the next layer's input, or postprocess.""" + stream = residual_batch.current(self.forward_batch) + if stream is not None: + residual = stream.residual if not isinstance(hidden_states, torch.Tensor): # A deferred MoE finalize handoff, consumed by the next prepare_attn. assert self.defer_moe_finalize, "unrequested deferred MoE handoff" - return hidden_states, residual - return self._complete(hidden_states, residual) + else: + hidden_states, residual = self._complete(hidden_states, residual) + return self.communicator._leave_ffn_output( + hidden_states, residual, stream, self._update + ) class MHCLayerCommunicator(LayerCommunicator): diff --git a/python/sglang/srt/layers/communicator/ops.py b/python/sglang/srt/layers/communicator/ops.py index 38a5958b24..3f1e62e009 100644 --- a/python/sglang/srt/layers/communicator/ops.py +++ b/python/sglang/srt/layers/communicator/ops.py @@ -320,6 +320,7 @@ def _mlp_input_dp_partial( gathers_residual: bool, places_cp_shards: bool = False, read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): """Attention DP: one rank adds the residual, the gather sums it, then read the input from the sum. With ``places_cp_shards`` each CP rank puts its @@ -384,6 +385,22 @@ def _tp_all_gather_scattered_rows( return output +def _dispatch_consumer( + hidden_states, residual, forward_batch, norm, context, *, paths, update=ADD, **call +): + if residual is None: + # Enter/read-only paths do not execute an update. + prepare = paths[False] + else: + try: + prepare = paths[update.adds_plainly] + except KeyError: + raise RuntimeError("producer update has no bound input path") from None + return prepare( + hidden_states, residual, forward_batch, norm, context, update=update, **call + ) + + def _consumer_step( hidden_states: Union[torch.Tensor, UnreducedOutput, HandoffOutput], residual: Optional[torch.Tensor], @@ -392,8 +409,12 @@ def _consumer_step( context: CommunicateContext, *, step: Callable, + adds_plainly: bool, carried_fusions: Tuple[Callable, ...], owes_by_construction: bool = False, + written_step: Optional[Callable] = None, + written: bool = False, + update: StageUpdate = ADD, **call, ): """A boundary's half into a stage: complete what the value carries, a sum @@ -404,6 +425,10 @@ def _consumer_step( that owes its sum by construction (``owes_by_construction``) is that sum's only carrier. ``call`` is what the stage's read takes (the attention's ``quant_format`` and ``post_residual_addition``).""" + if residual is not None and update.adds_plainly != adds_plainly: + raise RuntimeError("producer update does not match the boundary's capability") + if written and written_step is not None: + step = written_step if not isinstance(hidden_states, torch.Tensor): owed = hidden_states if residual is None: @@ -429,7 +454,9 @@ def _consumer_step( hidden_states = reduce_output(owed) else: hidden_states = owed.partial - return step(hidden_states, residual, forward_batch, norm, context, **call) + return step( + hidden_states, residual, forward_batch, norm, context, update=update, **call + ) def _read_input( @@ -442,7 +469,7 @@ def _read_input( layer_input: Optional[Callable], enters_stack: bool, read: StageRead, - update: StageUpdate, + update: StageUpdate = ADD, quant_format: str = "", post_residual_addition: Optional[torch.Tensor] = None, ): @@ -535,6 +562,7 @@ def _mlp_input_residual_into_sum( context: CommunicateContext, *, read: StageRead = NORM_READ, + update: StageUpdate = ADD, ): return _tp_all_reduce_with_scattered_residual( hidden_states, residual, layernorm, context, read @@ -565,12 +593,13 @@ def _mlp_input_gather_attention_cp( context: CommunicateContext, *, gather: Callable, + update: StageUpdate = ADD, ): """DSA and MLA CP: complete this rank's shard, then gather the shards, of equal length, over the attention-CP group. The residual stays on the shard.""" hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, context + hidden_states, residual, forward_batch, layernorm, context, update=update ) return dsa_cp_gather_hidden_states(hidden_states), residual @@ -583,6 +612,7 @@ def _mlp_input_gather_moe_cp( context: CommunicateContext, *, gather: Callable, + update: StageUpdate = ADD, ): """Gather for the FFN, then over the MoE-CP group so each rank holds all tokens of its MoE group (moe_dp_size < attn_cp_size). The residual stays at @@ -596,7 +626,7 @@ def _mlp_input_gather_moe_cp( return hidden_states, residual hidden_states, residual = gather( - hidden_states, residual, forward_batch, layernorm, context + hidden_states, residual, forward_batch, layernorm, context, update=update ) rows = moe_cp_gathered_rows(forward_batch) diff --git a/python/sglang/srt/layers/communicator/residual/__init__.py b/python/sglang/srt/layers/communicator/residual/__init__.py index 810267c3f3..c06f608f82 100644 --- a/python/sglang/srt/layers/communicator/residual/__init__.py +++ b/python/sglang/srt/layers/communicator/residual/__init__.py @@ -31,6 +31,9 @@ class StageUpdate(Protocol): # The stage writes its output into the residual itself at its end, instead # of leaving that to the next stage's read. at_producer: bool + # Its parameters and state remain valid after leaving the producer layer. + # Stateful or offloaded implementations must not opt in without that guarantee. + can_defer_across_layers: bool def update(self, hidden_states, residual) -> torch.Tensor: """The residual with the output written into it.""" diff --git a/python/sglang/srt/layers/communicator/residual/access.py b/python/sglang/srt/layers/communicator/residual/access.py index 70b38ee456..6e2408557d 100644 --- a/python/sglang/srt/layers/communicator/residual/access.py +++ b/python/sglang/srt/layers/communicator/residual/access.py @@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.output import ( UnreducedOutput, reduce_output, ) +from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors @@ -32,6 +33,8 @@ def buffer( """Storage that can be reused after prepare consumes the input. A finalize handoff has no reusable layer-output tensor. This does not complete or read the value held in the storage.""" + if isinstance(hidden_states, OwedOutput): + return hidden_states.contribution.value if isinstance(hidden_states, UnreducedOutput): return hidden_states.partial if isinstance(hidden_states, HandoffOutput): @@ -42,18 +45,35 @@ def buffer( def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple: """Complete the output before adding an extra contribution exactly once. Leave its residual update for the next prepare call.""" - hidden_states = reduce_output(hidden_states) + hidden_states = ( + residual.complete(hidden_states) + if isinstance(residual, ResidualStream) + else reduce_output(hidden_states) + ) hidden_states.add_(extra) return hidden_states, residual -def fold(hidden_states, residual) -> Tuple[torch.Tensor, None]: +def fold(hidden_states, residual): """Finish a plain layer output and fold its residual into a complete value. The following stage receives it with no outstanding residual addition.""" + stream = residual if isinstance(residual, ResidualStream) else None + if stream is not None: + hidden_states, residual = stream.finish(hidden_states) hidden_states = reduce_output(hidden_states) if residual is not None: hidden_states = hidden_states + residual - return hidden_states, None + return ( + (stream.write(hidden_states), stream) + if stream is not None + else (hidden_states, None) + ) + + +def written(hidden_states, forward_batch): + """Re-enter after a computation that already updated the full residual.""" + forward_batch.residual_stream = ResidualStream(hidden_states) + return hidden_states, forward_batch.residual_stream def finish_layer_stack( @@ -68,6 +88,13 @@ def finish_layer_stack( pipeline rank, or any other consumer outside the layers. A final norm that does a producer's handoff together with its own work (``final_norm_takes_handoff``) receives it as it is.""" + if isinstance(residual, ResidualStream): + stream = forward_batch.residual_stream + if residual is not stream: + raise RuntimeError("residual alias belongs to a different invocation") + output = stream.finish(hidden_states, takes_handoff=final_norm_takes_handoff) + forward_batch.residual_stream = None + return output if final_norm_takes_handoff and isinstance(hidden_states, HandoffOutput): return hidden_states, residual return reduce_output(hidden_states), residual @@ -99,6 +126,8 @@ def snapshot( ) -> torch.Tensor: """Copy a complete output with its plain residual. A statically declared sum is reduced on a copy; the main output and residual remain unchanged.""" + if isinstance(residual, ResidualStream): + return residual.snapshot(hidden_states, group=group) if group is not None: hidden_states = group.all_reduce(hidden_states.clone()) return hidden_states.clone() if residual is None else hidden_states + residual diff --git a/python/sglang/srt/layers/communicator/residual/add_norm.py b/python/sglang/srt/layers/communicator/residual/add_norm.py index 60e560f3f5..a55b88f05e 100644 --- a/python/sglang/srt/layers/communicator/residual/add_norm.py +++ b/python/sglang/srt/layers/communicator/residual/add_norm.py @@ -267,6 +267,7 @@ class Add: adds_plainly = True at_producer = False + can_defer_across_layers = True def update(self, hidden_states, residual): hidden_states += residual diff --git a/python/sglang/srt/layers/communicator/residual/batch.py b/python/sglang/srt/layers/communicator/residual/batch.py new file mode 100644 index 0000000000..9057f9b240 --- /dev/null +++ b/python/sglang/srt/layers/communicator/residual/batch.py @@ -0,0 +1,18 @@ +"""Own one invocation's residual stream, including independent TBO children.""" + +from sglang.srt.layers.communicator.residual.stream import ResidualStream + + +def start(forward_batch): + """Begin a fresh embedding-side invocation and return its stream.""" + stream = ResidualStream() + forward_batch.residual_stream = stream + return stream + + +def current(forward_batch): + """Return the invocation's authoritative stream while its stack is active.""" + stream = forward_batch.residual_stream + if stream is None: + raise RuntimeError("start the layer stack before entering a stage") + return stream diff --git a/python/sglang/srt/layers/communicator/residual/mhc.py b/python/sglang/srt/layers/communicator/residual/mhc.py index a21d5ea12e..3a42c0bb59 100644 --- a/python/sglang/srt/layers/communicator/residual/mhc.py +++ b/python/sglang/srt/layers/communicator/residual/mhc.py @@ -143,6 +143,7 @@ class _AttentionUpdate: adds_plainly = False at_producer = False + can_defer_across_layers = False def __init__(self, state: MHCState): self.state = state @@ -185,6 +186,7 @@ class _FfnUpdate: adds_plainly = False at_producer = True + can_defer_across_layers = False def __init__(self, state: MHCState): self.state = state diff --git a/python/sglang/srt/layers/communicator/residual/stream.py b/python/sglang/srt/layers/communicator/residual/stream.py new file mode 100644 index 0000000000..da7936eaa1 --- /dev/null +++ b/python/sglang/srt/layers/communicator/residual/stream.py @@ -0,0 +1,168 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""One forward's residual and its producer's outstanding contribution.""" + +from typing import Callable, Optional, Union + +import msgspec +import torch + +from sglang.srt.distributed import GroupCoordinator +from sglang.srt.layers.communicator.output import HandoffOutput, UnreducedOutput +from sglang.srt.layers.communicator.residual import StageUpdate + + +class CarriedSum(msgspec.Struct, frozen=True): + """Work left by the producer, including a move back to local token rows.""" + + group: Optional[GroupCoordinator] = None + reduce_and_redistribute: Optional[Callable] = None + + def complete(self, value): + if self.reduce_and_redistribute is not None: + return self.reduce_and_redistribute(value) + return self.group.all_reduce(value) + + +class Contribution(msgspec.Struct): + """The sole owner of an output's value, update and remaining work.""" + + value: Optional[torch.Tensor] + update: StageUpdate + owed: Union[CarriedSum, HandoffOutput, None] = None + + def for_boundary(self): + # These forms are private inputs to the existing fused-kernel adapters. + if isinstance(self.owed, CarriedSum): + return UnreducedOutput( + self.value, self.owed.group, self.owed.reduce_and_redistribute + ) + if isinstance(self.owed, HandoffOutput): + return self.owed + return self.value + + def complete(self): + if isinstance(self.owed, CarriedSum): + value = self.owed.complete(self.value) + elif isinstance(self.owed, HandoffOutput): + value = self.owed.complete() + else: + return self.value + self.value, self.owed = value, None + return value + + +class OwedOutput(msgspec.Struct, frozen=True): + """Opaque model-facing handle. Only its boundary may read the contribution.""" + + contribution: Contribution + + +class ResidualStream: + """Uninitialized, written, or holding one pending producer contribution. + + Each forward or TBO microbatch owns its stream. A prepare consumes its + pending contribution once; completing a sum for capture leaves the update + pending. No layout or batch selection is stored on the value. + """ + + __slots__ = ("residual", "pending") + + def __init__(self, residual=None): + self.residual = residual + self.pending = None + + @classmethod + def arrive(cls, hidden, residual, update): + """Rebuild an explicit tensor handoff, including each TBO microbatch.""" + stream = cls(residual) + if residual is None: + return stream.write(hidden), stream + return stream.leave(hidden, update), stream + + def write(self, residual): + self.residual = residual + self.pending = None + return residual + + def leave(self, output, update): + if self.pending is not None: + raise RuntimeError("cannot replace an unconsumed producer contribution") + if isinstance(output, UnreducedOutput): + contribution = Contribution( + output.partial, + update, + CarriedSum(output.group, output.reduce_and_redistribute), + ) + elif isinstance(output, HandoffOutput): + contribution = Contribution(None, update, output) + else: + contribution = Contribution(output, update) + self.pending = contribution + return OwedOutput(contribution) if contribution.owed is not None else output + + def check(self, hidden): + pending = self.pending + if pending is None: + if isinstance(hidden, OwedOutput): + raise RuntimeError("an owed handle has no contribution in this stream") + if self.residual is not None and hidden is not self.residual: + raise RuntimeError( + "written residual does not match the supplied output" + ) + return + if pending.owed is not None: + valid = isinstance(hidden, OwedOutput) and hidden.contribution is pending + else: + valid = hidden is pending.value + if not valid: + raise RuntimeError( + "output does not belong to this residual stream; " + "change layer outputs through boundary accessors" + ) + + def input(self, hidden): + self.check(hidden) + if self.pending is not None: + return self.pending.for_boundary(), self.residual + # An initialized, written stream needs read only. An uninitialized + # stream still uses the first stage's enter operation. + return hidden, None + + def complete(self, hidden): + self.check(hidden) + return self.pending.complete() if self.pending is not None else hidden + + def snapshot(self, hidden, *, group=None): + self.check(hidden) + pending = self.pending + if pending is None: + return hidden.clone() + if pending.owed is None: + value = pending.value + elif isinstance(pending.owed, CarriedSum): + value = pending.owed.complete(pending.value.clone()) + else: + raise NotImplementedError("a finalize handoff requires main-output capture") + if group is not None: + value = group.all_reduce(value.clone()) + return value.clone() if self.residual is None else value + self.residual + + def finish(self, hidden, *, takes_handoff=False): + self.check(hidden) + if self.pending is None: + return hidden, None + if takes_handoff and isinstance(self.pending.owed, HandoffOutput): + return self.pending.owed, self.residual + return self.pending.complete(), self.residual diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 71dd85d226..4c5c9f2d8e 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -71,6 +71,7 @@ from sglang.srt.utils.common import ceil_align, is_pin_memory_available if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.cp.base import BaseContextParallelMetadata from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata from sglang.srt.layers.logits_processor import LogitsProcessorOutput @@ -564,6 +565,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): dp_prefill_cuda_graph_max_prefix_len: int = 0 global_forward_mode: Optional[ForwardMode] = None + # Owned only while this invocation executes its local decoder stack. + residual_stream: Optional[ResidualStream] = None + # For two-batch overlap tbo_split_seq_index: Optional[int] = None diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index a8e4645419..aa940b6f0c 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -814,12 +815,12 @@ class BailingMoEModel(nn.Module): hidden_states = self.word_embeddings(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index d5ebb4c895..464b6c9b8e 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -19,6 +19,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -980,12 +981,12 @@ class BailingMoELinearModel(nn.Module): hidden_states = self.word_embeddings(input_ids) else: hidden_states = inputs_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) total_num_layers = self.end_layer - self.start_layer device = inputs_embeds.device if inputs_embeds is not None else input_ids.device diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 90bc8a57c6..6d2e4aacb4 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -26,6 +26,7 @@ import torch from torch import nn from transformers import PretrainedConfig +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear @@ -161,7 +162,7 @@ class BailingMoEModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) if self.is_hybrid: device = input_ids.device zero_allocator = BumpAllocator( diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 9e3dc8b56b..7abf968a22 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -1339,12 +1340,12 @@ class BailingMoELinearModel(nn.Module): hidden_states = self.word_embeddings(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) total_num_layers = self.end_layer - self.start_layer device = hidden_states.device diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index adc7374bfa..2fc88e189d 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -28,6 +28,7 @@ from sglang.kernels.ops.layernorm.fused_eh_norm import fused_eh_norm from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -215,7 +216,7 @@ class DeepseekModelNextN(nn.Module): else: hidden_states = self.eh_proj(eh_input) - residual = None + residual = residual_batch.start(forward_batch) index_topk_share = IndexTopKShareState.from_mtp_carry(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): hidden_states, residual, topk_indices = self.decoder( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index f6aaec7548..e109c6a915 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -76,6 +76,7 @@ from sglang.srt.layers.communicator import ( from sglang.srt.layers.communicator.adapters.context_parallel import ( maybe_prefetch_next_full_attention_kv, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx from sglang.srt.layers.dcp.planner import ( prepare_decode_context_parallel_metadata, @@ -3027,13 +3028,13 @@ class DeepseekV2Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) initial_topk_indices = None else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) initial_topk_indices = pp_proxy_tensors.tensors.get("topk_indices") index_topk_share = IndexTopKShareState(forward_batch, initial_topk_indices) if not self.pp_group.is_first_rank: diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 241edfcdb7..f35a7019dc 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -62,6 +62,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -1765,12 +1766,12 @@ class Dots3Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) normal_start_layer = self.start_layer normal_end_layer = self.end_layer diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index c5f431d2ad..c057321ff0 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -8,6 +8,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ReplicatedLinear @@ -113,7 +114,7 @@ class Dot3NoteModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): hidden_states, residual = head.decoder( positions, hidden_states, forward_batch, residual, zero_allocator diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 2a8e298bd4..225476c58f 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -12,6 +12,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -421,7 +422,7 @@ class FalconH1Model(nn.Module): else: hidden_states = self.embed_tokens(input_ids) * self.embedding_multiplier - residual = None + residual = residual_batch.start(forward_batch) for i in range(len(self.layers)): layer = self.layers[i] hidden_states, residual = layer( diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 90116925b4..8f88179448 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -24,6 +24,7 @@ from transformers import PretrainedConfig import sglang.srt.models.deepseek_v2 as deepseek_v2 from sglang.srt.configs.gigachat35 import GigaChat35Config from sglang.srt.layers.communicator import get_attn_tp_context +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -558,12 +559,12 @@ class GigaChat35Model(nn.Module): hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) total_num_layers = self.end_layer - self.start_layer zero_allocator = BumpAllocator( diff --git a/python/sglang/srt/models/gigachat35_mtp.py b/python/sglang/srt/models/gigachat35_mtp.py index 04ae77c264..fd98c8f8f2 100644 --- a/python/sglang/srt/models/gigachat35_mtp.py +++ b/python/sglang/srt/models/gigachat35_mtp.py @@ -18,6 +18,7 @@ from typing import Iterable, Optional import torch from torch import nn +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -106,7 +107,7 @@ class GigaChat35ModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) hidden_states, residual = self.decoder( positions=positions, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index de40514159..1551531c93 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -43,6 +43,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, is_dp_attention_enabled, @@ -1084,12 +1085,12 @@ class Glm4MoeModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) normal_start_layer = self.start_layer normal_end_layer = self.end_layer diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index ca97c0343e..379a5db7f0 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -38,6 +38,7 @@ from sglang.srt.layers.communicator import ( enable_moe_dense_fully_dp, get_attn_tp_context, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, is_dp_attention_enabled, @@ -776,12 +777,12 @@ class Glm4MoeLiteModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index c93d607cc7..2aded48fe7 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -22,6 +22,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor @@ -114,7 +115,7 @@ class Glm4MoeLiteModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): hidden_states, residual = self.decoder( positions, hidden_states, forward_batch, residual, zero_allocator diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index f742e4b732..f7f11f6511 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -22,6 +22,7 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor @@ -100,7 +101,7 @@ class Glm4MoeModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): hidden_states, residual = self.decoder( positions, hidden_states, forward_batch, residual diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 6dfd8a69e4..10b9ea6d92 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -32,6 +32,7 @@ from sglang.srt.layers.communicator import ( enable_moe_dense_fully_dp, get_attn_tp_context, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelBatchedLinear, @@ -1100,12 +1101,12 @@ class Glm5NextModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) device = hidden_states.device zero_allocator = BumpAllocator( buffer_size=total_num_layers * 2 * (2 if forward_batch.can_run_tbo else 1), diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index e1574b589b..110bdcf360 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -33,6 +33,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -700,12 +701,12 @@ class GptOssModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) # Capture hidden-state boundaries: boundary 0 is the embedding output, # and boundary i + 1 is the output after transformer block i. diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index f0514b5323..6b49b2e81a 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -11,6 +11,7 @@ from sglang.srt.configs.interns2_mobius import ( InternS2MobiusTextConfig, ) from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( @@ -762,7 +763,7 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): hidden_states = ( self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) - residual = None + residual = residual_batch.start(forward_batch) aux_hidden_states = [] for layer_idx, layer in enumerate(self.layers): hidden_states, residual = layer( diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 7c419ed4d5..9f464268c9 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -23,6 +23,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -581,12 +582,12 @@ class LagunaModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 6aec5f5ded..22cbdcb75b 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -36,6 +36,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -746,12 +747,12 @@ class LLaDA2MoeModel(nn.Module): hidden_states = self.word_embeddings(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) for i in range(self.start_layer, self.end_layer): with get_global_expert_distribution_recorder().with_current_layer(i): diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index 47736bda68..12ccfdc81d 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -26,6 +26,7 @@ from torch import nn from transformers import Llama4TextConfig from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -518,7 +519,7 @@ class Llama4Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) aux_hidden_states = [] for i in range(len(self.layers)): if i in self.layers_to_capture: @@ -561,6 +562,39 @@ class Llama4ForCausalLM(LlamaForCausalLM): ): super().__init__(config, quant_config, prefix) + @torch.no_grad() + def forward_split_prefill( + self, input_ids, positions, forward_batch, split_interval, input_embeds=None + ): + start, end = split_interval + if start == 0: + forward_batch.residual = residual_batch.start(forward_batch) + forward_batch.hidden_states = ( + self.model.embed_tokens(input_ids) + if input_embeds is None + else input_embeds + ) + for layer in self.model.layers[start:end]: + forward_batch.hidden_states, forward_batch.residual = layer( + positions, + forward_batch.hidden_states, + forward_batch, + forward_batch.residual, + ) + if end != self.model.config.num_hidden_layers: + return None + hidden, residual = LayerCommunicator.finish_layer_stack( + forward_batch.hidden_states, forward_batch.residual, forward_batch + ) + forward_batch.residual = None + if residual is None: + forward_batch.hidden_states = self.model.norm(hidden) + else: + forward_batch.hidden_states, _ = self.model.norm(hidden, residual) + return self.logits_processor( + input_ids, forward_batch.hidden_states, self.lm_head, forward_batch + ) + def get_input_embeddings(self): return self.model.embed_tokens diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 4693507d05..eb0bf2f40f 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -52,6 +52,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -624,7 +625,7 @@ class LongcatFlashModel(nn.Module): else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) aux_hidden_states = [] topk_indices = None diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 314712f366..373e223e57 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -42,6 +42,7 @@ from sglang.srt.configs import LongcatFlashConfig from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -269,7 +270,7 @@ class LongcatFlashModelNextN(nn.Module): ) ) - residual = None + residual = residual_batch.start(forward_batch) with get_global_expert_distribution_recorder().disable_this_region(): hidden_states, residual = self.decoder( positions, hidden_states, forward_batch, residual, zero_allocator diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 86b41e048e..02af86b11c 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -37,6 +37,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1037,12 +1038,12 @@ class MiMoV2Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = AuxHiddenStatePacker(len(self.layers_to_capture)) if forward_batch.can_run_tbo and not self.layers_to_capture: diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index c0c7676966..d0a638ffea 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -26,6 +26,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -223,7 +224,7 @@ class MiMoV2ModelNextN(nn.Module): positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=None, + residual=residual_batch.start(forward_batch), ) hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index e95b5122ad..2164d98c43 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -34,6 +34,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( attn_tp_all_reduce, is_dp_attention_enabled, @@ -1141,12 +1142,12 @@ class MiniMaxM2Model(nn.Module): hidden_states = self.get_input_embeddings(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] if forward_batch.can_run_tbo: diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index e59169e29e..15edb41322 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -40,6 +40,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1476,12 +1477,12 @@ class MiniMaxM3Model(nn.Module): hidden_states = embeds(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] if forward_batch.can_run_tbo: diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 0607b5dcb4..4d6ae2cea1 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -23,6 +23,7 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -1109,7 +1110,7 @@ class MossVLTextModel(nn.Module): vision_position_ids: Optional[torch.Tensor] = None, ) -> torch.Tensor: hidden_states = self.embed_tokens(input_ids) - residual = None + residual = residual_batch.start(forward_batch) for decoder_layer in self.layers: if isinstance(decoder_layer, MossVLCrossAttentionDecoderLayer): @@ -1126,6 +1127,9 @@ class MossVLTextModel(nn.Module): positions=positions, vision_position_ids=vision_position_ids, ) + hidden_states, residual = LayerCommunicator.written( + hidden_states, forward_batch + ) elif isinstance(decoder_layer, MossVLSelfAttentionDecoderLayer): hidden_states, residual = decoder_layer( positions=positions, diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index d0e5e30746..a63c59a472 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -31,6 +31,7 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -478,7 +479,7 @@ class NemotronHAttnLikeDecoderLayer(nn.Module): hidden_states = self._forward_mixer( hidden_states, forward_batch, mixer_exit.skips_reduction ) - return mixer_exit.finish(hidden_states), residual + return mixer_exit.finish(hidden_states, residual) class NemotronHMambaDecoderLayer(NemotronHAttnLikeDecoderLayer): @@ -732,12 +733,12 @@ class NemotronHModel(nn.Module): hidden_states = inputs_embeds else: hidden_states = self.embed_tokens(input_ids) - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] for i in range(self.start_layer, self.end_layer): diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 3e88bd2581..162b394689 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -18,6 +18,7 @@ import torch from torch import nn from sglang.srt.configs import NemotronHConfig +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear @@ -304,6 +305,8 @@ class NemotronHMultiTokenPredictor(nn.Module): residual = None for i in range(self.pattern_len): + if residual is None: + residual = residual_batch.start(forward_batch) hidden_states, residual = self.layers[str(i)]( inputs_embeds=inputs_embeds, hidden_states=hidden_states, diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index d1e96bf135..59f8089b24 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1175,12 +1176,12 @@ class Qwen2MoeModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] if forward_batch.can_run_tbo: @@ -1306,6 +1307,7 @@ class Qwen2MoeForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + forward_batch.residual = residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -1332,6 +1334,7 @@ class Qwen2MoeForCausalLM(nn.Module): hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual ) + forward_batch.residual = None forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index cdee95cd2c..735ec82428 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -1,11 +1,13 @@ # Adapted from qwen2.py import logging -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import torch from torch import nn from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import access as residual_access +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -446,6 +448,63 @@ class Qwen3Model(Qwen2Model): alt_stream=alt_stream, ) + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + ) -> Union[torch.Tensor, PPProxyTensors]: + + if self.pp_group.is_first_rank: + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + residual = residual_batch.start(forward_batch) + else: + assert pp_proxy_tensors is not None + hidden_states, residual = self.layers[ + self.start_layer + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) + + aux_hidden_states = [] + for i in range(self.start_layer, self.end_layer): + if i in self.layers_to_capture: + aux_hidden_states.append( + residual_access.snapshot(hidden_states, residual) + ) + layer = self.layers[i] + hidden_states, residual = layer( + positions, + hidden_states, + forward_batch, + residual, + ) + + hidden_states, residual = residual_access.finish_layer_stack( + hidden_states, residual, forward_batch + ) + if not self.pp_group.is_last_rank: + return PPProxyTensors( + { + "hidden_states": hidden_states, + "residual": residual, + } + ) + else: + if hidden_states.shape[0] != 0: + if residual is None: + hidden_states = self.norm(hidden_states) + else: + hidden_states, _ = self.norm(hidden_states, residual) + + if len(aux_hidden_states) == 0: + return hidden_states + + return hidden_states, aux_hidden_states + class Qwen3ForCausalLM(nn.Module): # BitandBytes specific attributes @@ -554,6 +613,7 @@ class Qwen3ForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + forward_batch.residual = residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -578,6 +638,7 @@ class Qwen3ForCausalLM(nn.Module): hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual ) + forward_batch.residual = None forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 5df575e712..f2a7c29cd2 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -30,8 +30,6 @@ from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( qwen3_5_gdn_prefill_projection_views, ) from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul - -# Configs from sglang.srt.configs.qwen3_5 import ( Qwen3_5Config, Qwen3_5MoeConfig, @@ -47,6 +45,9 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) + +# Configs +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1817,12 +1818,12 @@ class Qwen3_5ForCausalLM(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] # Pass through decoder layers diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 2c9363de64..ba1c21aa31 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -30,6 +30,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -1084,6 +1085,7 @@ class Qwen3MoeForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + forward_batch.residual = residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -1110,6 +1112,7 @@ class Qwen3MoeForCausalLM(nn.Module): hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual ) + forward_batch.residual = None forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 0806421f2b..28522c9ecd 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -16,6 +16,7 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -929,7 +930,7 @@ class Qwen3NextModel(nn.Module): else: hidden_states = self.embed_tokens(input_ids) - residual = None + residual = residual_batch.start(forward_batch) aux_hidden_states = [] for i in range(len(self.layers)): layer = self.layers[i] diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 2701e7dfd9..df6f19ed56 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -37,6 +37,7 @@ from sglang.srt.layers.attention.vision import ( VisionAttentionMetadata, prepare_vision_attention_metadata, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv3dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -1204,12 +1205,12 @@ class Qwen3LLMModel(Qwen3Model): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] for layer_idx, layer in enumerate( diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index eba3915927..2b6f941ae7 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -24,6 +24,7 @@ import torch.nn as nn from sglang.srt.configs.qwen3_vl import Qwen3VLMoeConfig, Qwen3VLMoeTextConfig from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import get_layer_id @@ -94,12 +95,12 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) aux_hidden_states = [] for layer_idx, layer in enumerate( diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index b3cfa3cf77..6f2e14886d 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -24,6 +24,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -1151,12 +1152,12 @@ class SarvamMLAModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] @@ -1267,7 +1268,7 @@ class SarvamMLAForCausalLM(nn.Module): forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: forward_batch.hidden_states = input_embeds - forward_batch.residual = None + forward_batch.residual = residual_batch.start(forward_batch) for i in range(start, end): with get_global_expert_distribution_recorder().with_current_layer(i): @@ -1291,6 +1292,7 @@ class SarvamMLAForCausalLM(nn.Module): hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual ) + forward_batch.residual = None forward_batch.hidden_states = hidden_states return self.logits_processor( input_ids, forward_batch.hidden_states, self.lm_head, forward_batch @@ -1455,7 +1457,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM): forward_batch.hidden_states = self.model.word_embeddings(input_ids) else: forward_batch.hidden_states = input_embeds - forward_batch.residual = None + forward_batch.residual = residual_batch.start(forward_batch) for i in range(start, end): with get_global_expert_distribution_recorder().with_current_layer(i): @@ -1479,6 +1481,7 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM): hidden_states, _ = self.model.norm( forward_batch.hidden_states, forward_batch.residual ) + forward_batch.residual = None forward_batch.hidden_states = hidden_states return self.logits_processor( diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 98d71a637c..ea7f7f50c8 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -12,6 +12,7 @@ from transformers import PretrainedConfig from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -403,12 +404,14 @@ class SDARModel(nn.Module): hidden_states = ( self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors, allow_missing_residual=True) + ].layer_communicator.from_pp( + pp_proxy_tensors, forward_batch, allow_missing_residual=True + ) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index baefae74c6..9479df3d6d 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -17,6 +17,7 @@ from sglang.srt.layers.communicator import ( LayerCommunicator, LayerFacts, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -469,12 +470,12 @@ class SDARMoeModel(nn.Module): hidden_states = ( self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 18087becc3..ead440abfc 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -23,6 +23,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.conv import Conv2dLayer from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, @@ -471,7 +472,7 @@ class Step3TextModel(nn.Module): else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) for i in range(len(self.layers)): layer = self.layers[i] hidden_states, residual = layer( diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 69769dfc2f..228d364d0b 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -12,6 +12,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerFacts +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) @@ -697,12 +698,12 @@ class Step3p5Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) for i in range(self.start_layer, self.end_layer): layer = self.layers[i] diff --git a/python/sglang/srt/models/step3p5_mtp.py b/python/sglang/srt/models/step3p5_mtp.py index ef882ed478..07356a85c1 100644 --- a/python/sglang/srt/models/step3p5_mtp.py +++ b/python/sglang/srt/models/step3p5_mtp.py @@ -6,6 +6,7 @@ import torch import torch.nn as nn from transformers import PretrainedConfig +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -113,7 +114,7 @@ class Step3p5AMultiTokenPredictor(nn.Module): positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=None, + residual=residual_batch.start(forward_batch), ) hidden_states, residual = self.mtp_block.layer_communicator.finish_layer_stack( hidden_states, residual, forward_batch diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index ea53dee826..f51da451e0 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -44,6 +44,7 @@ from sglang.srt.layers.communicator import ( LayerFacts, enable_moe_dense_fully_dp, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -1637,12 +1638,12 @@ class XllmModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual = residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None hidden_states, residual = self.layers[ self.start_layer - ].layer_communicator.from_pp(pp_proxy_tensors) + ].layer_communicator.from_pp(pp_proxy_tensors, forward_batch) for i in range(self.start_layer, self.end_layer): ctx = ( diff --git a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py index de61383765..f7e82d48da 100644 --- a/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py +++ b/test/registered/unit/batch_overlap/test_tbo_unreduced_input.py @@ -9,7 +9,8 @@ from unittest.mock import patch import torch from sglang.srt.batch_overlap import two_batch_overlap as tbo -from sglang.srt.layers.communicator import Layout, UnreducedOutput +from sglang.srt.layers.communicator import ADD, Layout, UnreducedOutput +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.utils import empty_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -18,6 +19,78 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") class TestTboEntryReducesItsInput(CustomTestCase): + def test_each_microbatch_owns_its_contribution_and_merge_keeps_the_update(self): + group = SimpleNamespace(all_reduce=lambda x: x * 2) + residual = torch.ones(4, 3) + stream = ResidualStream(residual) + hidden = stream.leave(UnreducedOutput(torch.ones(4, 3), group=group), ADD) + parent_batch = SimpleNamespace(residual_stream=stream) + parts_seen = [] + + def split(hidden_states, residual, **kwargs): + return [ + dict( + hidden_states=hidden_states[a:b], + residual=residual[a:b], + forward_batch=SimpleNamespace( + tbo_parent_token_range=(a, b), residual_stream=None + ), + ) + for a, b in ((0, 2), (2, 4)) + ] + + def execute(inputs_arr, **kwargs): + self.assertIsNone(parent_batch.residual_stream) + parts_seen.extend(inputs_arr) + self.assertIsNot(inputs_arr[0]["residual"], inputs_arr[1]["residual"]) + for part in inputs_arr: + state = part["residual"] + self.assertIs(state, part["forward_batch"].residual_stream) + value = part["hidden_states"] + self.assertIsNot(state.pending, stream.pending) + self.assertIs(state.pending.update, ADD) + with self.assertRaises(RuntimeError): + state.input( + inputs_arr[1 if part is inputs_arr[0] else 0]["hidden_states"] + ) + value, old_residual = state.input(value) + torch.testing.assert_close(value, torch.full((2, 3), 2.0)) + state.write(value + old_residual) + part["hidden_states"] = state.leave(value * 5, ADD) + return inputs_arr + + with ( + patch.object(tbo, "_model_forward_tbo_split_inputs", split), + patch.object(tbo, "execute_overlapped_operations", execute), + patch.object( + tbo.deep_gemm_wrapper, + "configure_deep_gemm_num_sms", + lambda _: empty_context(), + ), + ): + merged, output = tbo._model_forward_tbo( + inputs=dict( + hidden_states=hidden, + residual=stream, + positions=None, + forward_batch=parent_batch, + zero_allocator=None, + ), + operations_strategy=SimpleNamespace( + deep_gemm_num_sms=None, operations=[], tbo_delta_stages=0 + ), + layer_input_rows=Layout(frozenset()), + ) + self.assertEqual(len(parts_seen), 2) + self.assertIs(parent_batch.residual_stream, output) + self.assertTrue( + all(part["forward_batch"].residual_stream is None for part in parts_seen) + ) + self.assertIs(output.pending.update, ADD) + self.assertIs(output.pending.value, merged) + torch.testing.assert_close(merged, torch.full((4, 3), 10.0)) + torch.testing.assert_close(output.residual, torch.full((4, 3), 3.0)) + def test_split_and_merge_see_the_reduced_tensor(self): # Under attention DP the partial sum spans every DP rank's tokens; its # reduction also brings it back to this rank's three. @@ -52,7 +125,7 @@ class TestTboEntryReducesItsInput(CustomTestCase): hidden_states=hidden_states, residual=torch.zeros(3, 4), positions=None, - forward_batch=None, + forward_batch=SimpleNamespace(residual_stream=None), zero_allocator=None, ), operations_strategy=SimpleNamespace( diff --git a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py index a9ab72fb50..b99fc9df96 100644 --- a/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layers/moe/test_cutedsl_ar_fusion.py @@ -23,6 +23,7 @@ from sglang.srt.layers.communicator.fusions.cutedsl import ( prepare_cutedsl_fusion, ) from sglang.srt.layers.communicator.ops import _consumer_step, _read_input +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.flashinfer_mnnvl_cutedsl import ( FlashInferMNNVLCuteDSLARFusion, _retargeted_config, @@ -42,6 +43,12 @@ _MODULE = "sglang.srt.layers.communicator.fusions.cutedsl" _DECODE = SimpleNamespace(forward_mode=ForwardMode.DECODE, input_ids=torch.zeros(8)) +def _owned_input(hidden, residual, batch=_DECODE): + stream = residual_batch.start(batch) + stream.write(residual) + return stream.leave(hidden, ADD), stream, batch + + def _communicator(): comm = LayerCommunicator.__new__(LayerCommunicator) comm.fusions = CuteDSLFusion() @@ -57,6 +64,7 @@ def _communicator(): attention=SimpleNamespace( prepare=partial( _consumer_step, + adds_plainly=True, step=partial( _read_input, layer_input=None, @@ -130,7 +138,9 @@ def test_last_layer_consumes_but_does_not_skip_the_pending_all_reduce(eligible): lambda *a, **k: pytest.fail("fell through to the unfused path"), ), ): - out_hidden, _ = last.prepare_attn(hidden_states, torch.zeros(8, 8), _DECODE) + out_hidden, _ = last.prepare_attn( + *_owned_input(hidden_states, torch.zeros(8, 8)) + ) assert torch.equal(out_hidden, torch.ones(8, 8)) defer, absorb, _ = last._ffn_exit_fusions @@ -404,10 +414,10 @@ def test_a_handoff_the_kernel_does_not_take_is_completed_then_normed(): patch.object(CuteDSLFusion, "_common_eligible", return_value=False), ): hidden, residual = comm.prepare_attn( - _handoff(finish), torch.full((2, 8), 2.0), _DECODE + *_owned_input(_handoff(finish), torch.full((2, 8), 2.0)) ) finish.assert_called_once_with() - torch.testing.assert_close(residual, torch.full((2, 8), 3.0)) + torch.testing.assert_close(residual.residual, torch.full((2, 8), 3.0)) torch.testing.assert_close(hidden, torch.full((2, 8), 6.0)) @@ -417,7 +427,7 @@ def test_the_layer_stack_hands_a_handoff_only_to_a_final_norm_that_takes_it(): finish = MagicMock(return_value=torch.ones(2, 8)) handoff = _handoff(finish) hidden, _ = comm.finish_layer_stack( - handoff, torch.zeros(2, 8), _DECODE, final_norm_takes_handoff=takes + *_owned_input(handoff, torch.zeros(2, 8)), final_norm_takes_handoff=takes ) assert finish.call_count == completed assert (hidden is handoff) == takes diff --git a/test/registered/unit/layers/test_batch_owned_residual.py b/test/registered/unit/layers/test_batch_owned_residual.py new file mode 100644 index 0000000000..2b9e3682ee --- /dev/null +++ b/test/registered/unit/layers/test_batch_owned_residual.py @@ -0,0 +1,261 @@ +"""Residual ownership across layer execution and batch reuse.""" + +import unittest +import weakref +from contextlib import nullcontext +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.batch_overlap.two_batch_overlap import TboForwardBatchPreparer +from sglang.srt.layers.communicator import ( + ADD, + BoundarySteps, + EdgeDecl, + LayerCommunicator, + Layout, + StageEntry, + StageInput, + StageOutput, + make_boundary, +) +from sglang.srt.layers.communicator.layer import StageCommunicator +from sglang.srt.layers.communicator.output import UnreducedOutput +from sglang.srt.layers.communicator.residual import batch +from sglang.srt.model_executor.forward_batch_info import ( + ForwardBatch, + ForwardMode, + PPProxyTensors, +) +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestBatchOwnedResidual(CustomTestCase): + def test_start_releases_previous_call_and_resets_state(self): + fb = SimpleNamespace(residual_stream=None) + with self.assertRaisesRegex(RuntimeError, "start"): + batch.current(fb) + old = batch.start(fb) + self.assertIs(old, batch.current(fb)) + value = torch.ones(2, 4) + ref = weakref.ref(value) + old.write(value) + del value, old + fresh = batch.start(fb) + self.assertIs(fresh, batch.current(fb)) + self.assertIsNone(fresh.pending) + self.assertIsNone(fresh.residual) + self.assertIsNone(ref()) + + def test_terminal_completion_releases_batch_before_final_norm(self): + fb = SimpleNamespace(residual_stream=None) + stream = batch.start(fb) + residual = torch.full((2, 4), 3.0) + partial = torch.ones(2, 4) + group = SimpleNamespace(all_reduce=Mock(side_effect=lambda x: x * 2)) + stream.write(residual) + hidden = stream.leave(UnreducedOutput(partial, group=group), ADD) + complete, residual_value = LayerCommunicator.finish_layer_stack( + hidden, stream, fb + ) + self.assertIsNone(fb.residual_stream) + self.assertIs(residual_value, residual) + group.all_reduce.assert_called_once() + torch.testing.assert_close(complete + residual_value, torch.full((2, 4), 5.0)) + with self.assertRaisesRegex(RuntimeError, "start"): + batch.current(fb) + + def test_pp_receive_attaches_and_export_releases_the_same_stream(self): + communicator = LayerCommunicator.__new__(LayerCommunicator) + fb = SimpleNamespace(residual_stream=None) + hidden = torch.full((2, 4), 2.0) + residual = torch.ones(2, 4) + received, stream = communicator.from_pp( + PPProxyTensors({"hidden_states": hidden, "residual": residual}), fb + ) + self.assertIs(stream, batch.current(fb)) + self.assertIs(stream.pending.value, received) + self.assertIs(stream.pending.update, ADD) + exported, exported_residual = communicator.finish_layer_stack( + received, stream, fb + ) + self.assertIs(exported, hidden) + self.assertIs(exported_residual, residual) + self.assertIsNone(fb.residual_stream) + + def test_tbo_metadata_and_reused_batch_remain_stream_free_between_calls(self): + size = 4 + parent = ForwardBatch( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=size, + input_ids=torch.zeros(size, dtype=torch.long), + positions=torch.zeros(size, dtype=torch.long), + out_cache_loc=torch.zeros(size, dtype=torch.long), + req_pool_indices=torch.zeros(size, dtype=torch.long), + seq_lens=torch.ones(size, dtype=torch.int32), + seq_lens_cpu=torch.ones(size, dtype=torch.int32), + seq_lens_sum=size, + spec_info=None, + ) + for _ in range(2): + self.assertIsNone(parent.residual_stream) + with ( + get_context().override_server_args( + attention_backend="fa3", moe_dense_tp_size=None + ), + get_parallel().override(attn_tp_size=1), + ): + children = [ + TboForwardBatchPreparer.filter_batch( + parent, + start_token_index=start, + end_token_index=start + 2, + start_seq_index=start, + end_seq_index=start + 2, + out_num_token_non_padded=torch.tensor(2), + out_num_token_non_padded_cpu=2, + ) + for start in (0, 2) + ] + self.assertTrue(all(child.residual_stream is None for child in children)) + owner = batch.start(parent) + self.assertTrue(all(child.residual_stream is None for child in children)) + hidden = owner.write(torch.ones(size, 4)) + LayerCommunicator.finish_layer_stack(hidden, owner, parent) + cloned = replace(parent) + self.assertIsNone(cloned.residual_stream) + self.assertIsNone(parent.residual_stream) + + def test_split_prefill_retains_then_releases_batch_owner(self): + from sglang.srt.models.llama4 import Llama4ForCausalLM + from sglang.srt.models.qwen2_moe import Qwen2MoeForCausalLM + from sglang.srt.models.qwen3 import Qwen3ForCausalLM + from sglang.srt.models.qwen3_moe import Qwen3MoeForCausalLM + from sglang.srt.models.sarvam_moe import ( + SarvamMLAForCausalLM, + SarvamMoEForCausalLM, + ) + + for cls in ( + Llama4ForCausalLM, + Qwen3ForCausalLM, + Qwen3MoeForCausalLM, + Qwen2MoeForCausalLM, + SarvamMLAForCausalLM, + SarvamMoEForCausalLM, + ): + with self.subTest(model=cls.__name__): + context = ( + nullcontext() + if cls in (Llama4ForCausalLM, Qwen3ForCausalLM) + else patch( + cls.__module__ + ".get_global_expert_distribution_recorder", + return_value=SimpleNamespace( + with_current_layer=lambda i: nullcontext() + ), + ) + ) + with context: + self._check_split_prefill(cls) + + def _check_split_prefill(self, cls): + owner_ids = [] + group = SimpleNamespace(all_reduce=Mock(side_effect=lambda x: x * 2)) + + class Layer: + layer_communicator = LayerCommunicator.__new__(LayerCommunicator) + + def __call__(self, positions, hidden, forward_batch, residual): + stream = batch.current(forward_batch) + self_outer.assertIs(residual, stream) + owner_ids.append(id(stream)) + hidden, old = stream.finish(hidden) + stream.write(hidden if old is None else hidden + old) + return stream.leave( + UnreducedOutput(torch.ones_like(hidden), group=group), ADD + ), stream + + self_outer = self + fb = SimpleNamespace( + residual_stream=None, residual=None, forward_mode=ForwardMode.DECODE + ) + + def norm(hidden, residual): + self.assertIsNone(fb.residual_stream) + return hidden + residual, residual + + model = SimpleNamespace( + config=SimpleNamespace(num_hidden_layers=4), + layers=[Layer() for _ in range(4)], + norm=norm, + ) + lm = SimpleNamespace( + model=model, + lm_head=None, + logits_processor=lambda ids, hidden, head, fb: hidden, + ) + value = torch.ones(2, 4) + result = cls.forward_split_prefill(lm, None, None, fb, (0, 2), value) + self.assertIsNone(result) + self.assertIsNotNone(fb.residual_stream) + result = cls.forward_split_prefill(lm, None, None, fb, (2, 4)) + self.assertEqual(len(set(owner_ids)), 1) + self.assertIsNone(fb.residual_stream) + self.assertIsNone(fb.residual) + self.assertEqual(group.all_reduce.call_count, 4) + torch.testing.assert_close(result, torch.full((2, 4), 9.0)) + + def test_interleaved_stages_share_only_their_own_batch_stream(self): + class Read: + norms_plainly = False + + def enter(self, value): + return value + + def read(self, value, norm, quant_format="", **kwargs): + return value * 2, value + + def update_and_read(self, update, value, residual, norm, **kwargs): + updated = update.update(value, residual) + return updated * 2, updated + + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl(StageOutput(rows), StageInput(rows, read=Read()), rows, rows) + ) + steps = BoundarySteps( + StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False + ) + stage = StageCommunicator( + SimpleNamespace(input_layernorm=None, _context=None), + "attention", + "input_layernorm", + ) + a, b = [SimpleNamespace(residual_stream=None) for _ in range(2)] + sa, sb = batch.start(a), batch.start(b) + first, first_alias = stage.prepare(torch.ones(2, 4), sa, a, steps) + second, second_alias = stage.prepare(torch.full((2, 4), 10.0), sb, b, steps) + self.assertIs(first_alias, batch.current(a)) + self.assertIs(second_alias, batch.current(b)) + first = sa.leave(first * 3, ADD) + second = sb.leave(second * 5, ADD) + torch.testing.assert_close( + stage.prepare(first, sa, a, steps)[0], torch.full((2, 4), 14.0) + ) + torch.testing.assert_close( + stage.prepare(second, sb, b, steps)[0], torch.full((2, 4), 220.0) + ) + self.assertIsNot(batch.current(a), batch.current(b)) + batch.start(a) + self.assertIsNone(batch.current(a).residual) + self.assertIsNotNone(batch.current(b).residual) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/test_boundary_edges.py b/test/registered/unit/layers/test_boundary_edges.py index 9ff78ef6bd..9d44762531 100644 --- a/test/registered/unit/layers/test_boundary_edges.py +++ b/test/registered/unit/layers/test_boundary_edges.py @@ -26,6 +26,7 @@ from sglang.srt.layers.communicator import ( make_output_boundary, ) from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -324,7 +325,48 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): other = make_boundary(self.edge(written)).prepare.keywords["step"] self.assertIs(other.func, comm_ops._mlp_input_dp_replicate) self.assertTrue(other.keywords["reduces_attention_tp"]) - self.assertIs(other.keywords["update"], written) + self.assertNotIn("update", other.keywords) + + def test_capabilities_prebind_both_orders_without_capturing_an_update(self): + edge = msgspec.structs.replace( + self.edge(None), update_capabilities=(True, False) + ) + boundary = make_boundary(edge) + paths = boundary.prepare.keywords["paths"] + self.assertEqual(set(paths), {True, False}) + self.assertIs(paths[True].keywords["step"].func, comm_ops._mlp_input_dp_partial) + self.assertIs( + paths[False].keywords["step"].func, comm_ops._mlp_input_dp_replicate + ) + for path in paths.values(): + self.assertNotIn("update", path.keywords["step"].keywords) + + def test_actual_update_selects_a_prebound_path(self): + full = rows(sizes(tp=1)) + edge = EdgeDecl( + StageOutput(full, update=None), + StageInput(full), + full, + full, + update_capabilities=(True, False), + ) + boundary = make_boundary(edge) + + def norm(value, residual=None): + return value if residual is None else (value + residual, value + residual) + + for update, expected in ((comm.ADD, 4.0), (_WrittenIn(), 5.0)): + with self.subTest(plain=update.adds_plainly): + hidden, residual = boundary.prepare( + torch.ones(2, 4), + torch.full((2, 4), 3.0), + None, + norm, + None, + update=update, + ) + torch.testing.assert_close(hidden, torch.full((2, 4), expected)) + torch.testing.assert_close(residual, hidden) def test_fused_kernels_take_only_a_plain_add(self): full = rows(sizes(tp=2)) @@ -404,6 +446,27 @@ class TestTheProducersUpdateChoosesTheOrder(CustomTestCase): with self.assertRaises(NotImplementedError): make_output_boundary(edge) + def test_cross_layer_update_requires_a_lifetime_guarantee(self): + local = Layout(frozenset()) + stateful_update = SimpleNamespace(adds_plainly=False, at_producer=False) + edge = EdgeDecl( + StageOutput(local, update=stateful_update), StageInput(local), local, local + ) + with self.assertRaisesRegex(NotImplementedError, "lifetime"): + make_output_boundary(edge) + + def test_pipeline_cannot_reconstruct_a_non_add_update(self): + local = Layout(frozenset()) + update = SimpleNamespace( + adds_plainly=False, at_producer=False, can_defer_across_layers=True + ) + edge = EdgeDecl( + StageOutput(local, update=update), StageInput(local), local, local + ) + with get_parallel().override(pp_size=2): + with self.assertRaisesRegex(NotImplementedError, "pipeline boundaries"): + make_output_boundary(edge) + class ProbeRead: """A read that marks what it reads: the input is the residual plus 100. diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index 630d32a35b..23fbec4b6a 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -23,6 +23,7 @@ from sglang.srt.layers.communicator import ( reduce_output, sequence_parallel_layer_sides, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator @@ -130,6 +131,15 @@ def make_communicator( return communicator +def finish_exit(ffn_exit, hidden, residual): + """Inspect the selected completion form inside a batch-owned stream.""" + stream = residual_batch.start(ffn_exit.forward_batch) + stream.write(residual) + hidden, output_stream = ffn_exit.finish(hidden, stream) + assert output_stream is residual_batch.current(ffn_exit.forward_batch) + return output_stream.input(hidden) + + def published_flags(): forward = get_forward() return forward.fuse_mlp_allreduce, forward.mlp_reduce_scatter @@ -139,13 +149,13 @@ class TestFfnExit(CustomTestCase): def setUp(self): self.hidden_states = torch.ones(3, 4) self.residual = torch.zeros(3, 4) - self.forward_batch = object() + self.forward_batch = types.SimpleNamespace(residual_stream=None) def run_exit(self, communicator): with communicator.ffn_exit(self.forward_batch) as ffn_exit: seen = published_flags() hidden_states = self.hidden_states * 2 - return seen, ffn_exit.finish(hidden_states, self.residual) + return seen, finish_exit(ffn_exit, hidden_states, self.residual) def test_reduction_left_to_next_layer_is_unreduced(self): communicator = make_communicator(fuse=True, reduce_scatter=False) @@ -230,7 +240,7 @@ class TestFfnExit(CustomTestCase): decide = communicator._ffn_sum_moves_to_next_layer decide.return_value = not fuse hidden_states = self.hidden_states * 2 - hidden_states, _ = ffn_exit.finish(hidden_states, self.residual) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, self.residual) self.assertEqual(seen, (fuse, False)) self.assertEqual(isinstance(hidden_states, UnreducedOutput), fuse) self.assertEqual(communicator._complete_ffn_output_now.called, not fuse) @@ -286,13 +296,13 @@ class TestFfnExit(CustomTestCase): seen = published_flags() + (get_forward().defer_moe_finalize,) self.assertEqual(seen, (True, False, True)) self.assertEqual( - ffn_exit.finish(handoff, self.residual), (handoff, self.residual) + finish_exit(ffn_exit, handoff, self.residual), (handoff, self.residual) ) communicator._complete_ffn_output_now.assert_not_called() self.assertFalse(get_forward().defer_moe_finalize) # The MoE may decline per forward; a tensor then takes the fused exit. - hidden_states, _ = ffn_exit.finish(self.hidden_states * 2, self.residual) + hidden_states, _ = finish_exit(ffn_exit, self.hidden_states * 2, self.residual) self.assertIsInstance(hidden_states, UnreducedOutput) @@ -306,9 +316,11 @@ class TestReduceOutput(CustomTestCase): def test_reduction_left_by_the_last_layer_runs_once(self): communicator = self.communicator(fuse=True) - with communicator.ffn_exit(object()) as ffn_exit: + with communicator.ffn_exit( + types.SimpleNamespace(residual_stream=None) + ) as ffn_exit: hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, torch.zeros(3, 4)) hidden_states = reduce_output(hidden_states) self.all_reduce.assert_called_once() @@ -331,9 +343,11 @@ class TestReduceOutput(CustomTestCase): def test_complete_hidden_states_pass_through(self): communicator = self.communicator(fuse=False) - with communicator.ffn_exit(object()) as ffn_exit: + with communicator.ffn_exit( + types.SimpleNamespace(residual_stream=None) + ) as ffn_exit: hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, torch.zeros(3, 4)) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, torch.zeros(3, 4)) self.assertIs(reduce_output(hidden_states), hidden_states) self.assertIsNone(reduce_output(None)) @@ -345,9 +359,11 @@ class TestReduceOutput(CustomTestCase): self.all_reduce.reset_mock() communicator = self.communicator(fuse=fuse) residual = torch.zeros(3, 4) - with communicator.ffn_exit(object()) as ffn_exit: + with communicator.ffn_exit( + types.SimpleNamespace(residual_stream=None) + ) as ffn_exit: hidden_states = torch.ones(3, 4) - hidden_states, _ = ffn_exit.finish(hidden_states, residual) + hidden_states, _ = finish_exit(ffn_exit, hidden_states, residual) hidden_states, residual_out = communicator.finish_layer_stack( hidden_states, residual, object() diff --git a/test/registered/unit/layers/test_declared_attention_cp.py b/test/registered/unit/layers/test_declared_attention_cp.py index a009c0f6a2..d96af6c7c3 100644 --- a/test/registered/unit/layers/test_declared_attention_cp.py +++ b/test/registered/unit/layers/test_declared_attention_cp.py @@ -18,6 +18,7 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator.adapters import context_parallel as dsa_cp +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.communicator_patch import patch_communicator from sglang.test.test_utils import CustomTestCase @@ -61,6 +62,13 @@ def group(name, ranks): return SimpleNamespace(name=name, ranks=list(ranks)) +def prepare_mlp(communicator, hidden, residual, forward_batch): + forward_batch.residual_stream = ResidualStream(residual) + return communicator.prepare_mlp( + hidden, forward_batch.residual_stream, forward_batch + ) + + class TestAttentionCpBoundary(CustomTestCase): def setUp(self): generator = torch.Generator().manual_seed(0) @@ -200,15 +208,21 @@ class TestAttentionCpBoundary(CustomTestCase): with self.as_rank( cp, dict(gather=record_gather(cp), reduce_scatter=unused) ): - self.build(allow_reduce_scatter).prepare_mlp( - self.values[cp], self.residuals[cp], self.cp_extend() + prepare_mlp( + self.build(allow_reduce_scatter), + self.values[cp], + self.residuals[cp], + self.cp_extend(), ) gathered, residuals = {}, {} for cp in range(CP_SIZE): with self.as_rank(cp, dict(gather=fill_gather, reduce_scatter=unused)): - gathered[cp], residuals[cp] = self.build( - allow_reduce_scatter - ).prepare_mlp(self.values[cp], self.residuals[cp], self.cp_extend()) + gathered[cp], residuals[cp] = prepare_mlp( + self.build(allow_reduce_scatter), + self.values[cp], + self.residuals[cp], + self.cp_extend(), + ) expected_rows = torch.cat( [self.values[cp] + self.residuals[cp] for cp in range(CP_SIZE)] ) @@ -245,10 +259,12 @@ class TestAttentionCpBoundary(CustomTestCase): cp, dict(gather=fill_gather, reduce_scatter=reduce_scatter) ) as rank: communicator = self.build(allow_reduce_scatter) - with communicator.ffn_exit(self.cp_extend()) as exit_: + batch = self.cp_extend() + batch.residual_stream = ResidualStream(residuals[cp].residual) + with communicator.ffn_exit(batch) as exit_: published[cp] = rank.flags.mlp_reduce_scatter output = ffn_output(cp, leaves=published[cp]) - back[cp], _ = exit_.finish(output, residuals[cp]) + back[cp], _ = exit_.finish(output, batch.residual_stream) return published, back, reduced def test_the_reduce_scatter_completes_the_sum_the_moe_leaves(self): diff --git a/test/registered/unit/layers/test_declared_decoder_boundary.py b/test/registered/unit/layers/test_declared_decoder_boundary.py index ca1cdf16a5..4f2cbf203c 100644 --- a/test/registered/unit/layers/test_declared_decoder_boundary.py +++ b/test/registered/unit/layers/test_declared_decoder_boundary.py @@ -23,12 +23,12 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers import layernorm_sp from sglang.srt.layers.communicator import ( + ADD, LayerCommunicator, LayerFacts, MHCLayerCommunicator, SumGroup, TokenAxis, - UnreducedOutput, ) from sglang.srt.layers.communicator import boundary as comm_boundary from sglang.srt.layers.communicator import ( @@ -41,7 +41,9 @@ from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.communicator import ( sequence_parallel_layer_sides, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.communicator.residual import mhc as mhc_module +from sglang.srt.layers.communicator.residual.stream import OwedOutput from sglang.srt.layers.moe import utils as moe_utils from sglang.srt.runtime_context import LoRABatchLayout from sglang.test.ci.ci_register import register_cpu_ci @@ -495,7 +497,7 @@ class TestMhcOnTheDeclarations(CustomTestCase): if "read" in step.keywords: # The FFN input: the attention output's hc_post, then the FFN's read. self.assertIs(step.keywords["read"], residual.ffn_read) - self.assertIs(step.keywords["update"], residual.attention_update) + self.assertNotIn("update", step.keywords) else: # The FFN output's move: its hc_post. self.assertIs(step.keywords["update"], residual.ffn_update) @@ -591,11 +593,12 @@ class TestMhcOnTheDeclarations(CustomTestCase): patch_communicator("_reduce_and_redistribute_output_step", choose), patch_communicator("_to_local_tokens", to_local_tokens), ): + stream = residual_batch.start(forward_batch) + stream.write(torch.full((2, 4), 2.0)) with communicator.ffn_exit(forward_batch) as ffn_exit: self.assertTrue(ffn_exit.mlp_reduce_scatter) - hidden, residual = ffn_exit.finish( - torch.ones(4, 4), torch.full((2, 4), 2.0) - ) + hidden, stream = ffn_exit.finish(torch.ones(4, 4), stream) + hidden, residual = stream.input(hidden) choose.assert_called_once() to_local_tokens.assert_called_once() self.assertIs(to_local_tokens.call_args.args[0], move) @@ -987,7 +990,7 @@ class TestTheAttentionOutputDecidesItsSum(CustomTestCase): residual=sides.input_rows, residual_to=sides.ffn_residual_rows, need=sides.ffn.input, - update=comm.ADD, + adds_plainly=True, fusions=(), force_layernorm_before_gather=force, residual_joins_sum=False, @@ -1112,7 +1115,7 @@ class TestFusedKernelsTakeOnlyTheStepsTheyComplete(CustomTestCase): residual=sides.input_rows, residual_to=sides.ffn_residual_rows, need=sides.ffn.input, - update=comm.ADD, + adds_plainly=True, fusions=fusions, force_layernorm_before_gather=False, residual_joins_sum=False, @@ -1192,7 +1195,7 @@ class TestTheSequenceParallelRegion(CustomTestCase): for step, read in ((ffn, comm.NORM_READ), (attention, comm.NORM_QUANT_READ)): self.assertIsNone(step.keywords["layer_input"]) self.assertIs(step.keywords["read"], read) - self.assertIs(step.keywords["update"], comm.ADD) + self.assertNotIn("update", step.keywords) replace = msgspec.structs.replace return replace( steps, @@ -1347,7 +1350,7 @@ class TestInputScatteredAttention(CustomTestCase): residual=local, residual_to=attention, need=comm.StageInput(attention), - update=comm.ADD, + adds_plainly=True, fusions=(), force_layernorm_before_gather=False, residual_joins_sum=joins, @@ -1367,7 +1370,7 @@ class TestInputScatteredAttention(CustomTestCase): residual=attention, residual_to=local, need=comm.StageInput(local), - update=comm.ADD, + adds_plainly=True, fusions=(), force_layernorm_before_gather=False, residual_joins_sum=False, @@ -1451,12 +1454,14 @@ class TestInputScatteredAttention(CustomTestCase): ), patch_communicator("get_forward", lambda: SimpleNamespace(sp_active=False)), ): - hidden, residual = communicator.prepare_attn( - torch.ones(4, HIDDEN), torch.full((4, HIDDEN), 3.0), None - ) + batch = SimpleNamespace(residual_stream=None) + stream = residual_batch.start(batch) + stream.write(torch.full((4, HIDDEN), 3.0)) + contribution = stream.leave(torch.ones(4, HIDDEN), ADD) + hidden, residual = communicator.prepare_attn(contribution, stream, batch) self.assertEqual(scattered, [4]) # Norm: (2 * (h + r), h + r) on the slice, with h the completed sum. - torch.testing.assert_close(residual, torch.full((2, HIDDEN), 5.0)) + torch.testing.assert_close(residual.residual, torch.full((2, HIDDEN), 5.0)) torch.testing.assert_close(hidden, torch.full((2, HIDDEN), 10.0)) def test_the_last_layer_completes_its_own_sum(self): @@ -1942,7 +1947,9 @@ class TestBranchRows(CustomTestCase): return value with patch_communicator("move_rows", recorded): - dense.branch_input(moe, "h0", "residual", None) + dense.branch_input( + moe, "h0", "residual", SimpleNamespace(residual_stream=None) + ) self.assertEqual( moves, [ @@ -2320,6 +2327,7 @@ class TestTwoLayers(CustomTestCase): ) def forward(rank): + batch = SimpleNamespace(**vars(forward_batch), residual_stream=None) s = states[rank] layers = [ LayerCommunicator( @@ -2338,13 +2346,13 @@ class TestTwoLayers(CustomTestCase): ] hidden = torch.zeros(s.local_rows, HIDDEN).double() hidden[: s.rows] = embeddings[s.dp] - residual = None + residual = residual_batch.start(batch) handed_on = [] for layer_index, layer in enumerate(layers): - hidden, residual = layer.prepare_attn(hidden, residual, forward_batch) + hidden, residual = layer.prepare_attn(hidden, residual, batch) hidden = attention(hidden, s) - hidden, residual = layer.prepare_mlp(hidden, residual, forward_batch) - with layer.ffn_exit(forward_batch) as ffn_exit: + hidden, residual = layer.prepare_mlp(hidden, residual, batch) + with layer.ffn_exit(batch) as ffn_exit: if not sparse[layer_index]: hidden = dense_mlp(hidden, s) elif a2a: @@ -2354,9 +2362,7 @@ class TestTwoLayers(CustomTestCase): hidden = moe(hidden, s) hidden, residual = ffn_exit.finish(hidden, residual) handed_on.append(type(hidden)) - hidden, residual = layers[-1].finish_layer_stack( - hidden, residual, forward_batch - ) + hidden, residual = layers[-1].finish_layer_stack(hidden, residual, batch) # The last a2a layer folds the residual into its output. if residual is not None: residual = residual[: s.rows] @@ -2441,7 +2447,7 @@ class TestTwoLayers(CustomTestCase): ) self.assertIs( first_layer_hands_on, - UnreducedOutput if owes else torch.Tensor, + OwedOutput if owes else torch.Tensor, ) @@ -2528,7 +2534,10 @@ class TestALayerThatIsOneStage(CustomTestCase): ) def test_its_steps_run_the_declared_read_and_update(self): - read, update = self.ProbeRead(), SimpleNamespace(adds_plainly=True) + read, update = ( + self.ProbeRead(), + SimpleNamespace(adds_plainly=True, can_defer_across_layers=True), + ) communicator = build( layer_facts(1, 3), parallel_of(attn_dp=1, attn_tp=2), diff --git a/test/registered/unit/layers/test_dp_attention_cp_gather.py b/test/registered/unit/layers/test_dp_attention_cp_gather.py index c9cf5ebeb6..e6b8f1f7bb 100644 --- a/test/registered/unit/layers/test_dp_attention_cp_gather.py +++ b/test/registered/unit/layers/test_dp_attention_cp_gather.py @@ -15,6 +15,7 @@ import torch from sglang.srt.layers import communicator as comm from sglang.srt.layers import dp_attention, layernorm_sp from sglang.srt.layers.communicator import ops as comm_ops +from sglang.srt.layers.communicator.residual.stream import ResidualStream from sglang.srt.layers.cp import base as cp_base from sglang.srt.layers.cp import padding as cp_padding from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy @@ -125,6 +126,13 @@ def cp_replicated(forward_mode, rows, seed): return group +def prepare_mlp(communicator, hidden, residual, forward_batch): + forward_batch.residual_stream = ResidualStream(residual) + return communicator.prepare_mlp( + hidden, forward_batch.residual_stream, forward_batch + ) + + class TestDpCpGather(CustomTestCase): def run_ranks( self, @@ -278,8 +286,8 @@ class TestDpCpGather(CustomTestCase): steps = r.communicator._batch_steps(r.forward_batch) self.assertIs(steps.ffn.prepare.keywords["step"].func, ffn_input) hidden_states, residual = rank_inputs(*rank) - return r.communicator.prepare_mlp( - hidden_states, residual, r.forward_batch + return prepare_mlp( + r.communicator, hidden_states, residual, r.forward_batch ) # The all-reduce is a sum over every rank: record what each rank hands @@ -317,8 +325,9 @@ class TestDpCpGather(CustomTestCase): tokens = own.shape[0] hidden_states, residual = gather(rank, lambda x: summed.clone()) torch.testing.assert_close(hidden_states, expected, rtol=0, atol=0) - torch.testing.assert_close(residual[:tokens], own, rtol=0, atol=0) + torch.testing.assert_close(residual.residual[:tokens], own, rtol=0, atol=0) with as_rank(*rank, None) as r: + r.forward_batch.residual_stream = residual back, _ = r.communicator.postprocess_layer( 3 * hidden_states, residual, r.forward_batch ) diff --git a/test/registered/unit/layers/test_layernorm_sp.py b/test/registered/unit/layers/test_layernorm_sp.py index b3c0af90f4..9bbdf7c5bc 100644 --- a/test/registered/unit/layers/test_layernorm_sp.py +++ b/test/registered/unit/layers/test_layernorm_sp.py @@ -26,6 +26,7 @@ from sglang.srt.layers.communicator import ops as comm_ops from sglang.srt.layers.communicator import ( sequence_parallel_layer_sides, ) +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.runtime_context import ( get_flags, @@ -203,6 +204,7 @@ class TestSpRegionSteps(CustomTestCase): update=comm.ADD, ), carried_fusions=(), + adds_plainly=True, ), input_rows=comm.Layout(frozenset()), input_move=attention_input, @@ -217,8 +219,17 @@ class TestSpRegionSteps(CustomTestCase): ffn_sum_is_movable=True, ) + def owned_input(self, hidden, residual, mode=ForwardMode.EXTEND, *, record=True): + batch = SimpleNamespace(forward_mode=mode, residual_stream=None) + stream = residual_batch.start(batch) + if residual is not None: + stream.write(residual) + if record: + hidden = stream.leave(hidden, comm.ADD) + return hidden, stream, batch + def run_prepare_attn(self, communicator, mode, hidden, residual): - batch = SimpleNamespace(forward_mode=mode) + hidden, residual, batch = self.owned_input(hidden, residual, mode) with ( get_flags().sp.override(enabled=True), get_forward().scoped(sp_active=False), @@ -241,7 +252,7 @@ class TestSpRegionSteps(CustomTestCase): self.assertTrue(active) scatter.assert_called_once() torch.testing.assert_close(h, torch.full((1, 4), 2.0)) - torch.testing.assert_close(r, torch.ones(1, 4)) + torch.testing.assert_close(r.residual, torch.ones(1, 4)) def test_decode_leaves_the_region_closed(self): communicator = self.communicator(first_layer=True) @@ -277,17 +288,20 @@ class TestSpRegionSteps(CustomTestCase): return_value=SimpleNamespace(input_scattered=False), ): h, r = communicator.prepare_attn( - hidden, - residual.clone(), - SimpleNamespace(forward_mode=ForwardMode.EXTEND), + *self.owned_input(hidden, residual.clone()) ) torch.testing.assert_close(h, torch.full((1, 4), 8.0)) - h, r = communicator.prepare_mlp(hidden, residual.clone(), object()) + h, r = communicator.prepare_mlp( + *self.owned_input(hidden, residual.clone(), record=False) + ) torch.testing.assert_close(h, torch.full((1, 4), 8.0)) - torch.testing.assert_close(r, torch.full((1, 4), 4.0)) - out = communicator.postprocess_layer(hidden, residual, object()) + torch.testing.assert_close(r.residual, torch.full((1, 4), 4.0)) + batch = SimpleNamespace(residual_stream=None) + stream = residual_batch.start(batch) + stream.write(residual) + out = communicator.postprocess_layer(hidden, stream, batch) self.assertIs(out[0], hidden) - self.assertIs(out[1], residual) + self.assertIs(out[1], stream) if __name__ == "__main__": diff --git a/test/registered/unit/layers/test_prepare_attn_steps.py b/test/registered/unit/layers/test_prepare_attn_steps.py index 331cd9c995..a0d44d9ded 100644 --- a/test/registered/unit/layers/test_prepare_attn_steps.py +++ b/test/registered/unit/layers/test_prepare_attn_steps.py @@ -62,6 +62,7 @@ def communicator(norm): attention=comm.StageEntry( prepare=partial( comm_ops._consumer_step, + adds_plainly=True, step=partial( comm_ops._read_input, layer_input=None, @@ -131,6 +132,16 @@ def platform(*, use_aiter=False, gfx95=False, fusion=False, kernel_group=True): yield kernels, all_reduce, group +def prepare_attn(communicator, hidden, residual, forward_batch=None, **kwargs): + from sglang.srt.layers.communicator.residual.add_norm import ADD + from sglang.srt.layers.communicator.residual.stream import ResidualStream + + batch = forward_batch or SimpleNamespace() + batch.residual_stream = ResidualStream(residual) + hidden = batch.residual_stream.leave(hidden, ADD) + return communicator.prepare_attn(hidden, batch.residual_stream, batch, **kwargs) + + class TestPrepareAttnSteps(CustomTestCase): def test_each_quant_format_uses_its_kernel(self): for quant_format, flags, kernel in ( @@ -154,8 +165,12 @@ class TestPrepareAttnSteps(CustomTestCase): platform(**flags) as (kernels, _, _), ): norm = Norm() - communicator(norm).prepare_attn( - torch.ones(2, 4), residual, None, quant_format=quant_format + prepare_attn( + communicator(norm), + torch.ones(2, 4), + residual, + None, + quant_format=quant_format, ) for name, mock in kernels.items(): self.assertEqual(mock.called, name == kernel, name) @@ -170,10 +185,14 @@ class TestPrepareAttnSteps(CustomTestCase): ): with self.subTest(quant_format=quant_format), platform(**flags): hidden_states = torch.ones(2, 4) - _, residual = communicator(Norm()).prepare_attn( - hidden_states, None, None, quant_format=quant_format + _, residual = prepare_attn( + communicator(Norm()), + hidden_states, + None, + None, + quant_format=quant_format, ) - self.assertIs(residual, hidden_states) + self.assertIs(residual.residual, hidden_states) def test_a_pending_sum_is_completed_once(self): for fusion in (False, True): @@ -183,7 +202,7 @@ class TestPrepareAttnSteps(CustomTestCase): ): norm = Norm() hidden_states = comm.UnreducedOutput(torch.ones(2, 4), group=group) - communicator(norm).prepare_attn(hidden_states, torch.zeros(2, 4), None) + prepare_attn(communicator(norm), hidden_states, torch.zeros(2, 4), None) self.assertEqual(all_reduce.call_count, 0 if fusion else 1) self.assertEqual( norm.calls, ["fused_all_reduce_norm"] if fusion else ["norm"] @@ -192,7 +211,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_pending_sum_keeps_the_quant_format_without_fusion(self): with platform(use_aiter=True) as (kernels, all_reduce, group): norm = Norm() - communicator(norm).prepare_attn( + prepare_attn( + communicator(norm), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -205,7 +225,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_pending_sum_over_another_group_skips_the_fused_kernel(self): with platform(fusion=True, kernel_group=False) as (_, all_reduce, group): norm = Norm() - communicator(norm).prepare_attn( + prepare_attn( + communicator(norm), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -216,7 +237,8 @@ class TestPrepareAttnSteps(CustomTestCase): def test_a_post_residual_addition_bypasses_the_fused_kernel(self): with platform(fusion=True) as (_, all_reduce, group): norm = Norm() - communicator(norm).prepare_attn( + prepare_attn( + communicator(norm), comm.UnreducedOutput(torch.ones(2, 4), group=group), torch.zeros(2, 4), None, @@ -235,7 +257,8 @@ class TestPrepareAttnSteps(CustomTestCase): partial = torch.ones(3, 4) local = torch.full((local_rows, 4), 7.0) step = MagicMock(return_value=local) - hidden_states, residual = communicator(norm).prepare_attn( + hidden_states, residual = prepare_attn( + communicator(norm), comm.UnreducedOutput(partial, reduce_and_redistribute=step), torch.zeros(local_rows, 4), None, @@ -247,7 +270,7 @@ class TestPrepareAttnSteps(CustomTestCase): torch.testing.assert_close(hidden_states, local * 2) else: self.assertEqual(norm.calls, []) - self.assertIs(residual, local) + self.assertIs(residual.residual, local) def test_an_owed_reduce_scatter_is_not_offered_to_a_fused_kernel(self): with platform(fusion=True) as (_, all_reduce, _): @@ -259,6 +282,7 @@ class TestPrepareAttnSteps(CustomTestCase): c._steps.attention, prepare=partial( comm_ops._consumer_step, + adds_plainly=True, step=partial( comm_ops._read_input, layer_input=None, @@ -272,7 +296,8 @@ class TestPrepareAttnSteps(CustomTestCase): ) partial_sum = torch.ones(3, 4) step = MagicMock(return_value=torch.full((1, 4), 7.0)) - c.prepare_attn( + prepare_attn( + c, comm.UnreducedOutput(partial_sum, reduce_and_redistribute=step), torch.zeros(1, 4), None, @@ -289,7 +314,8 @@ class TestPrepareAttnSteps(CustomTestCase): ): norm = Norm() with self.assertRaises(RuntimeError): - communicator(norm).prepare_attn( + prepare_attn( + communicator(norm), comm.UnreducedOutput( torch.ones(2, 4), reduce_and_redistribute=step ), diff --git a/test/registered/unit/layers/test_residual_stream.py b/test/registered/unit/layers/test_residual_stream.py new file mode 100644 index 0000000000..b8e67139c1 --- /dev/null +++ b/test/registered/unit/layers/test_residual_stream.py @@ -0,0 +1,201 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + +import torch + +from sglang.srt.layers.communicator import ( + BoundarySteps, + EdgeDecl, + Layout, + StageEntry, + StageInput, + StageOutput, + make_boundary, +) +from sglang.srt.layers.communicator.layer import StageCommunicator +from sglang.srt.layers.communicator.output import UnreducedOutput +from sglang.srt.layers.communicator.residual.access import add_to_output +from sglang.srt.layers.communicator.residual.add_norm import ADD +from sglang.srt.layers.communicator.residual.stream import OwedOutput, ResidualStream +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestResidualStream(CustomTestCase): + def setUp(self): + self.group = SimpleNamespace(all_reduce=Mock(side_effect=lambda x: x.mul_(2))) + self.residual = torch.full((2, 4), 3.0) + self.partial = torch.ones(2, 4) + self.stream = ResidualStream(self.residual) + self.hidden = self.stream.leave( + UnreducedOutput(self.partial, group=self.group), ADD + ) + + def test_written_input_does_not_repeat_first_stage_initialization(self): + class Read: + norms_plainly = False + enters = 0 + + def enter(self, value): + self.enters += 1 + return value * 3 + + def read(self, value, norm, quant_format=""): + return value * 2, value + + read = Read() + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl(StageOutput(rows), StageInput(rows, read=read), rows, rows), + enters_stack=True, + ) + steps = BoundarySteps( + StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False + ) + stage = StageCommunicator( + SimpleNamespace(input_layernorm=None, _context=None), + "attention", + "input_layernorm", + ) + value = torch.ones(2, 4) + batch = SimpleNamespace(residual_stream=ResidualStream()) + hidden, stream = stage.prepare(value, batch.residual_stream, batch, steps) + self.assertEqual(read.enters, 1) + torch.testing.assert_close(hidden, value * 6) + hidden, stream = stage.prepare(stream.residual, stream, batch, steps) + self.assertEqual(read.enters, 1) + torch.testing.assert_close(hidden, value * 6) + + def test_consumer_executes_the_carried_update_not_the_bound_producers(self): + class Update: + adds_plainly = False + at_producer = False + + def __init__(self, increment): + self.increment = increment + self.calls = 0 + + def update(self, value, residual): + self.calls += 1 + return value + residual + self.increment + + class Read: + norms_plainly = False + + def update_and_read(self, update, value, residual, norm, **kwargs): + residual = update.update(value, residual) + return residual * 2, residual + + declared = Update(100) + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl( + StageOutput(rows, update=declared), + StageInput(rows, read=Read()), + rows, + rows, + ) + ) + steps = BoundarySteps( + StageEntry(boundary.prepare, rows), None, StageOutput(rows), None, False + ) + stage = StageCommunicator( + SimpleNamespace(input_layernorm=None, _context=None), + "attention", + "input_layernorm", + ) + for increment in (2, 7): + with self.subTest(increment=increment): + actual = Update(increment) + stream = ResidualStream(torch.full((2, 4), 3.0)) + hidden = stream.leave(torch.ones(2, 4), actual) + batch = SimpleNamespace(residual_stream=stream) + result, stream = stage.prepare(hidden, stream, batch, steps) + torch.testing.assert_close( + result, torch.full((2, 4), 2.0 * (4 + increment)) + ) + self.assertEqual(actual.calls, 1) + self.assertIsNone(stream.pending) + self.assertEqual(declared.calls, 0) + + def test_unbound_update_capability_fails_before_consuming_the_sum(self): + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl(StageOutput(rows), StageInput(rows), rows, rows) + ) + with self.assertRaisesRegex(RuntimeError, "capability"): + boundary.prepare( + self.partial, + self.residual, + None, + None, + None, + update=SimpleNamespace(adds_plainly=False), + ) + + def test_snapshot_finishes_a_copy_without_consuming_main_work(self): + snapshot = self.stream.snapshot(self.hidden) + torch.testing.assert_close(snapshot, torch.full((2, 4), 5.0)) + torch.testing.assert_close(self.partial, torch.ones(2, 4)) + torch.testing.assert_close(self.residual, torch.full((2, 4), 3.0)) + self.assertIsInstance(self.hidden, OwedOutput) + self.assertIsNotNone(self.stream.pending.owed) + with self.assertRaises(TypeError): + self.hidden + 1 + with self.assertRaises(TypeError): + self.hidden[0] + + def test_main_capture_completes_once_and_leaves_update_pending(self): + hidden = self.stream.complete(self.hidden) + self.assertIs(self.stream.complete(hidden), hidden) + self.group.all_reduce.assert_called_once() + self.assertIs(self.stream.pending.value, hidden) + self.assertIs(self.stream.pending.update, ADD) + self.assertIsNone(self.stream.pending.owed) + self.assertIs(self.stream.residual, self.residual) + torch.testing.assert_close(hidden, torch.full((2, 4), 2.0)) + with self.assertRaises(RuntimeError): + self.stream.input(self.hidden) + with self.assertRaises(RuntimeError): + self.stream.input(hidden.clone()) + + def test_deepstack_adds_once_after_completing_the_sum(self): + hidden, stream = add_to_output( + self.hidden, self.stream, torch.full((2, 4), 7.0) + ) + self.assertIs(stream, self.stream) + self.assertIs(stream.pending.value, hidden) + self.assertIs(stream.residual, self.residual) + torch.testing.assert_close(hidden, torch.full((2, 4), 9.0)) + self.group.all_reduce.assert_called_once() + + def test_handles_cannot_cross_microbatches_or_be_reused_after_take(self): + other = ResidualStream(self.residual.clone()) + other.leave(UnreducedOutput(self.partial.clone(), group=self.group), ADD) + with self.assertRaises(RuntimeError): + other.input(self.hidden) + with self.assertRaises(RuntimeError): + self.stream.leave(torch.zeros_like(self.partial), ADD) + self.stream.write(self.residual) + with self.assertRaises(RuntimeError): + self.stream.input(self.hidden) + self.assertEqual(self.stream.input(self.residual), (self.residual, None)) + + def test_local_row_move_is_owed_even_without_an_all_reduce(self): + move = Mock(side_effect=lambda value: value[:1]) + stream = ResidualStream(self.residual[:1]) + hidden = stream.leave( + UnreducedOutput(self.partial, reduce_and_redistribute=move), ADD + ) + self.assertIsInstance(hidden, OwedOutput) + completed = stream.complete(hidden) + self.assertEqual(completed.shape[0], 1) + self.assertIs(stream.pending.value, completed) + move.assert_called_once_with(self.partial) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/lora/test_triton_dp_attention_unit.py b/test/registered/unit/lora/test_triton_dp_attention_unit.py index 44765a022b..4475883ab9 100644 --- a/test/registered/unit/lora/test_triton_dp_attention_unit.py +++ b/test/registered/unit/lora/test_triton_dp_attention_unit.py @@ -17,6 +17,7 @@ from sglang.srt.layers.communicator import ( TokenAxis, ) from sglang.srt.layers.communicator.ops import _consumer_step, _read_input +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.layers.dp_attention import DpPaddingMode from sglang.srt.lora.backend.base_backend import BaseLoRABackend from sglang.srt.lora.backend.triton_backend import ( @@ -253,12 +254,16 @@ def test_communicator_publishes_layout_at_each_transition( update=ADD, ), carried_fusions=(), + adds_plainly=True, ), input_move=lambda hidden_states, **kwargs: hidden_states, handoff=lambda hidden_states, *args: hidden_states, ), ffn=SimpleNamespace( - prepare=lambda hidden_states, residual, *args: (hidden_states, residual), + prepare=lambda hidden_states, residual, *args, **kwargs: ( + hidden_states, + residual, + ), input_rows=gathered if gathered_over_dp else local, input_move=None, handoff=None, @@ -268,9 +273,13 @@ def test_communicator_publishes_layout_at_each_transition( hidden = torch.zeros(num_tokens, 4) with get_forward().scoped(lora_batch_layout=initial): for _ in range(2): - communicator.prepare_mlp(hidden, hidden, None) + batch = SimpleNamespace(residual_stream=None) + stream = residual_batch.start(batch) + stream.write(hidden) + communicator.prepare_mlp(hidden, stream, batch) assert get_forward().lora_batch_layout is expected_mlp - communicator.prepare_attn(hidden, None, None) + stream = residual_batch.start(batch) + communicator.prepare_attn(hidden, stream, batch) assert get_forward().lora_batch_layout is expected_attn diff --git a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py index 2dcf8d765f..ad08e8a454 100644 --- a/test/registered/unit/models/test_aux_capture_deferred_allreduce.py +++ b/test/registered/unit/models/test_aux_capture_deferred_allreduce.py @@ -108,15 +108,19 @@ class DeferringLayer(nn.Module): *args, **kwargs, ): + stream = forward_batch.residual_stream + assert residual is stream + hidden_states, residual = stream.input(hidden_states) hidden_states = comm.reduce_output(hidden_states) if residual is None: residual = hidden_states.clone() else: # later norms mutate the residual; captured snapshots must stay intact residual.add_(hidden_states) + stream.write(residual) with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: partial = torch.full_like(hidden_states, 0.5) - hidden_states, residual = ffn_exit.finish(partial, residual) + hidden_states, residual = ffn_exit.finish(partial, stream) if self.return_topk: return hidden_states, residual, None return hidden_states, residual @@ -241,26 +245,33 @@ class TestPipelineResidualReception(CustomTestCase): ) streams = torch.randn(2, 4, 3) hidden, residual = comm_instance.from_pp( - PPProxyTensors({"hidden_states": streams}) + PPProxyTensors({"hidden_states": streams}), + SimpleNamespace(residual_stream=None), ) self.assertIs(hidden, streams) - self.assertIsNone(residual) + self.assertIsNone(residual.pending) + self.assertIs(residual.residual, hidden) def test_optional_residual_and_declared_partial_keep_the_wire_values(self): comm_instance = comm.LayerCommunicator.__new__(comm.LayerCommunicator) partial = torch.randn(2, 4) prior = torch.randn_like(partial) hidden, residual = comm_instance.from_pp( - PPProxyTensors({"hidden_states": partial, "residual": prior}) + PPProxyTensors({"hidden_states": partial, "residual": prior}), + SimpleNamespace(residual_stream=None), ) self.assertIs(hidden, partial) - self.assertIs(residual, prior) + self.assertIs(residual.residual, prior) + self.assertIs(residual.pending.value, partial) missing = PPProxyTensors({"hidden_states": partial}) with self.assertRaises(KeyError): - comm_instance.from_pp(missing) - hidden, residual = comm_instance.from_pp(missing, allow_missing_residual=True) + comm_instance.from_pp(missing, SimpleNamespace(residual_stream=None)) + hidden, residual = comm_instance.from_pp( + missing, SimpleNamespace(residual_stream=None), allow_missing_residual=True + ) self.assertIs(hidden, partial) - self.assertIsNone(residual) + self.assertIsNone(residual.pending) + self.assertIs(residual.residual, hidden) if __name__ == "__main__": diff --git a/test/registered/unit/models/test_deepstack_deferred_allreduce.py b/test/registered/unit/models/test_deepstack_deferred_allreduce.py index f27cba6825..8790f9f4ac 100644 --- a/test/registered/unit/models/test_deepstack_deferred_allreduce.py +++ b/test/registered/unit/models/test_deepstack_deferred_allreduce.py @@ -9,6 +9,7 @@ import torch from torch import nn from sglang.srt.layers.communicator import LayerCommunicator, UnreducedOutput +from sglang.srt.layers.communicator.residual.add_norm import ADD from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -43,15 +44,16 @@ class DeferringLayer(nn.Module): def forward( self, positions=None, hidden_states=None, forward_batch=None, residual=None, **_ ): - if isinstance(hidden_states, UnreducedOutput): - hidden_states = all_reduce(hidden_states.partial) + stream = forward_batch.residual_stream + hidden_states, residual = stream.finish(hidden_states) residual = hidden_states if residual is None else hidden_states + residual - if self.is_last_layer: - return torch.ones_like(residual), residual - return ( - UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP), - residual, + stream.write(residual) + output = ( + torch.ones_like(residual) + if self.is_last_layer + else UnreducedOutput(torch.full_like(residual, 1 / TP_SIZE), group=GROUP) ) + return stream.leave(output, ADD), stream class SumNorm(nn.Module): @@ -112,7 +114,7 @@ class TestDeepstackOnDeferredReduction(CustomTestCase): return model.forward( input_ids=None, positions=None, - forward_batch=None, + forward_batch=SimpleNamespace(), input_embeds=self.embeds.clone(), input_deepstack_embeds=deepstack, ) @@ -175,7 +177,7 @@ class TestSplitPrefillCompletion(CustomTestCase): input_embeds=torch.zeros(tokens, HIDDEN), ) self.assertIsNone(first) - self.assertIsInstance(batch.hidden_states, UnreducedOutput) + self.assertIsNotNone(batch.residual_stream.pending.owed) result = model_cls.forward_split_prefill( wrapper, None, None, batch, (2, NUM_LAYERS) ) diff --git a/test/registered/unit/models/test_layer_stack_exit.py b/test/registered/unit/models/test_layer_stack_exit.py index 437650d4fd..d2babac9df 100644 --- a/test/registered/unit/models/test_layer_stack_exit.py +++ b/test/registered/unit/models/test_layer_stack_exit.py @@ -300,7 +300,7 @@ class TestLayerStackExit(CustomTestCase): def test_each_stack_ends_at_the_exit(self): problems = [] for name, (forward, layers) in sorted(self.complete_census.subjects().items()): - exits = list(calls(forward, EXIT)) + exits = list(calls(forward, EXIT)) + list(calls(forward, "take_output")) if not exits: if delegates_to_super(forward): continue @@ -342,6 +342,9 @@ class TestLayerStackExit(CustomTestCase): forbidden = { "UnreducedOutput", "HandoffOutput", + "OwedOutput", + "Contribution", + "ResidualStream", "reduce_output", "layer_input_buffer", } @@ -384,7 +387,9 @@ class TestLayerStackExit(CustomTestCase): if any(calls(forward, "forward_split_prefill")): continue # delegates to the inherited implementation with self.subTest(model=name, file=path.name): - exits = list(calls(forward, EXIT)) + exits = list(calls(forward, EXIT)) + list( + calls(forward, "take_output") + ) self.assertEqual(len(exits), 1) exit_call = exits[0] self.assertFalse(inside_loop(forward, exit_call)) @@ -397,7 +402,18 @@ class TestLayerStackExit(CustomTestCase): and item.test.left.id == "end" and any(part is exit_call for part in ast.walk(item)) ] - self.assertEqual(len(final_segments), 1) + early_returns = [ + item + for item in forward.body + if isinstance(item, ast.If) + and isinstance(item.test, ast.Compare) + and isinstance(item.test.left, ast.Name) + and item.test.left.id == "end" + and any(isinstance(op, ast.NotEq) for op in item.test.ops) + and any(isinstance(part, ast.Return) for part in item.body) + and item.end_lineno < exit_call.lineno + ] + self.assertEqual(len(final_segments) + len(early_returns), 1) for call in ast.walk(forward): if isinstance(call, ast.Call) and is_final_norm(call): self.assertLess(exit_call.lineno, call.lineno) diff --git a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py index 720bf76ad2..fbdea260db 100644 --- a/test/registered/unit/models/test_nemotron_h_mtp_reduction.py +++ b/test/registered/unit/models/test_nemotron_h_mtp_reduction.py @@ -5,6 +5,8 @@ from unittest.mock import Mock import torch from torch import nn +from sglang.srt.layers.communicator import ADD +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.models import nemotron_h_mtp from sglang.srt.runtime_context import get_context, get_flags, get_parallel @@ -75,17 +77,22 @@ class TestNemotronMTPReduction(CustomTestCase): partial = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) residual = torch.tensor([[7.0, 3.0], [5.0, 9.0]]) expected = partial * tp + residual + batch = SimpleNamespace( + forward_mode=ForwardMode.DECODE, + input_ids=torch.zeros(2, dtype=torch.long), + residual_stream=None, + ) + stream = residual_batch.start(batch) + stream.write(residual) + partial = stream.leave(partial, ADD) hidden, output_residual = layer( inputs_embeds=torch.zeros_like(partial), hidden_states=partial, - residual=residual, - forward_batch=SimpleNamespace( - forward_mode=ForwardMode.DECODE, - input_ids=torch.zeros(2, dtype=torch.long), - ), + residual=stream, + forward_batch=batch, ) torch.testing.assert_close(hidden, expected) - torch.testing.assert_close(output_residual, expected) + torch.testing.assert_close(output_residual.residual, expected) self.assertEqual(reduce.call_count, int(tp > 1)) diff --git a/test/registered/unit/models/test_nemotron_h_stages.py b/test/registered/unit/models/test_nemotron_h_stages.py index 4fdcc14c0a..dfa084823f 100644 --- a/test/registered/unit/models/test_nemotron_h_stages.py +++ b/test/registered/unit/models/test_nemotron_h_stages.py @@ -78,7 +78,9 @@ class TestStageEdges(CustomTestCase): into.produced.leaves_for_next_layer, produced.leaves_for_next_layer, ) - self.assertEqual(layers[0].edges[0].produced, StageOutput(rows)) + self.assertEqual( + layers[0].edges[0].produced, StageOutput(rows, update=None) + ) self.assertTrue(layers[0].enters_stack) self.assertFalse(any(layer.enters_stack for layer in layers[1:])) last = layers[-1].edges[1].produced @@ -104,11 +106,11 @@ class TestStageEdges(CustomTestCase): ) # Without attention TP a mixer's output is complete. into = stages("M-", tp=1)[1].edges[0].produced - self.assertEqual(into, StageOutput(into.layout)) + self.assertEqual(into, StageOutput(into.layout, update=None)) # A MoE on this rank's own rows hands on a complete output; an a2a # backend dispatches only the MoE, so an MLP still sums over TP. into = stages("EM", tp=2, a2a=True)[1].edges[0].produced - self.assertEqual(into, StageOutput(into.layout)) + self.assertEqual(into, StageOutput(into.layout, update=None)) into = stages("-M", tp=2, a2a=True)[1].edges[0].produced self.assertEqual( (into.group, into.always_leaves, into.leaves_for_next_layer), diff --git a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py index 2fcc3cb5ea..cc0c75284e 100644 --- a/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py +++ b/test/registered/unit/models/test_step3p5_dense_reduce_scatter.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock import torch from sglang.srt.layers import communicator as comm +from sglang.srt.layers.communicator.residual import batch as residual_batch from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -33,6 +34,11 @@ def dense_layer(*, reduce_scatter): communicator = SimpleNamespace( prepare_attn=lambda h, r, fb, **_: (h, h if r is None else r), prepare_mlp=lambda h, r, fb: (h, r), + _batch_steps=lambda fb: SimpleNamespace( + ffn_output=SimpleNamespace(update=comm.ADD) + ), + _declared_ffn_sum=lambda steps, skipped: None, + _leave_ffn_output=lambda h, r, stream, update, declared_sum=None: (h, r), # A dense layer never leaves its sum to the next layer. _select_ffn_completion=lambda fb: comm.FfnCompletion( defer_moe_finalize=False, @@ -54,11 +60,14 @@ class TestStep3p5DenseReduceScatter(CustomTestCase): for reduce_scatter in (False, True): with self.subTest(reduce_scatter=reduce_scatter): layer, seen, complete = dense_layer(reduce_scatter=reduce_scatter) + batch = SimpleNamespace(residual_stream=None) + stream = residual_batch.start(batch) + stream.write(torch.ones(2, 4)) layer.forward( positions=None, hidden_states=torch.ones(2, 4), - forward_batch=None, - residual=None, + forward_batch=batch, + residual=stream, ) self.assertEqual(seen["mlp_reduce_scatter"], reduce_scatter) self.assertFalse(seen["fuse_mlp_allreduce"])