diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 2e7a195bbd..a6851c6c15 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -19,7 +19,6 @@ from sglang.srt.layers.layer_boundary import ( tbo_split_moves, ) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch -from sglang.srt.layers.layer_boundary.residual.access import export_output from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.moe import ( get_deepep_mode, @@ -1101,18 +1100,15 @@ def _model_forward_filter_inputs( def _model_forward_tbo_merge_outputs(output_a, output_b, original_len): stream_a, stream_b = output_a["residual"], output_b["residual"] - has_stream = isinstance(stream_a, ResidualStream) - assert has_stream == isinstance(stream_b, ResidualStream) + pending_a, pending_b = stream_a.pending, stream_b.pending + assert (pending_a is None) == (pending_b is None) update = None - if has_stream: - pending_a, pending_b = stream_a.pending, stream_b.pending - assert (pending_a is None) == (pending_b is None) - if pending_a is not None: - assert pending_a.update is pending_b.update - update = pending_a.update + if pending_a is not None: + assert pending_a.update is pending_b.update + update = pending_a.update for output in (output_a, output_b): - output["hidden_states"], output["residual"] = export_output( - output["hidden_states"], output["residual"], output["forward_batch"] + output["hidden_states"], output["residual"] = output["residual"].export( + output["hidden_states"] ) def _handle_key(name): @@ -1133,11 +1129,7 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len): return res hidden, residual = _handle_key("hidden_states"), _handle_key("residual") - return ( - ResidualStream.from_handoff(hidden, residual, update) - if has_stream - else (hidden, residual) - ) + return ResidualStream.from_handoff(hidden, residual, update) # -------------------------------- Utilities and wrappers --------------------------------------- diff --git a/python/sglang/srt/layers/layer_boundary/residual/access.py b/python/sglang/srt/layers/layer_boundary/residual/access.py index 9210e6f132..3ab6e7e1bc 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/access.py +++ b/python/sglang/srt/layers/layer_boundary/residual/access.py @@ -17,14 +17,9 @@ from typing import Optional, Tuple, Union import torch -from sglang.srt.distributed import GroupCoordinator -from sglang.srt.layers.layer_boundary.output import ( - DeferredFinalize, - UnreducedOutput, - complete_owed, -) -from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.layers.layer_boundary.output import DeferredFinalize, UnreducedOutput +from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput +from sglang.srt.model_executor.forward_batch_info import PPProxyTensors def buffer( @@ -42,67 +37,6 @@ def buffer( return hidden_states -def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple: - """Complete the output before adding an extra contribution exactly once. - Leave its residual update for the next prepare call.""" - hidden_states = ( - residual.complete(hidden_states) - if isinstance(residual, ResidualStream) - else complete_owed(hidden_states) - ) - hidden_states.add_(extra) - return hidden_states, residual - - -def fold(hidden_states, residual): - """Finish a plain layer output and fold its residual into a complete value. - The following stage receives it with no outstanding residual addition.""" - stream = residual if isinstance(residual, ResidualStream) else None - if stream is not None: - if stream.pending is not None and not stream.pending.update.is_plain_add: - raise NotImplementedError("fold requires a plain residual update") - hidden_states, residual = stream.export(hidden_states) - hidden_states = complete_owed(hidden_states) - if residual is not None: - hidden_states = hidden_states + residual - return ( - (stream.write(hidden_states), stream) - if stream is not None - else (hidden_states, None) - ) - - -def written(hidden_states): - """Re-enter after a computation that already updated the full residual.""" - return hidden_states, ResidualStream(hidden_states) - - -def export_output( - hidden_states: Union[torch.Tensor, UnreducedOutput, DeferredFinalize], - residual: Optional[torch.Tensor], - forward_batch: ForwardBatch, - *, - final_norm_takes_handoff: bool = False, - preserve_declared: bool = False, -) -> Tuple[Union[torch.Tensor, DeferredFinalize], Optional[torch.Tensor]]: - """Complete what this layer left for a next layer, for callers that carry - an explicit residual tensor or merge TBO microbatches. Stage-boundary - decoders use residual_batch.final_norm(), to_pp() or take_output() instead: - to_pp() keeps a declared partial sum for the receiving from_pp(), which - this helper would complete early. A final norm that does a producer's - handoff together with its own work (``final_norm_takes_handoff``) - receives it as it is.""" - if isinstance(residual, ResidualStream): - return residual.export( - hidden_states, - takes_handoff=final_norm_takes_handoff, - preserve_declared=preserve_declared, - ) - if final_norm_takes_handoff and isinstance(hidden_states, DeferredFinalize): - return hidden_states, residual - return complete_owed(hidden_states), residual - - def final_norm_pair(hidden_states, residual, norm, capture=None, **read_kwargs): """The final norm and an optional capture of the same updated residual. @@ -136,20 +70,3 @@ def from_pp( else: residual = tensors["residual"] return tensors["hidden_states"], residual - - -def snapshot( - hidden_states: torch.Tensor, - residual: Optional[torch.Tensor], - *, - group: Optional[GroupCoordinator] = None, -) -> torch.Tensor: - """Copy a complete output with its plain residual. A statically declared - sum is reduced on a copy; the main output and residual remain unchanged.""" - if isinstance(residual, ResidualStream): - if group is not None: - raise ValueError("a residual stream carries its own reduction") - return residual.snapshot(hidden_states) - if group is not None: - hidden_states = group.all_reduce(hidden_states.clone()) - return hidden_states.clone() if residual is None else hidden_states + residual diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index 3bd56baab5..8ebb21e505 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -13,6 +13,7 @@ # ============================================================================== """Residual access for one layer-stack invocation or TBO microbatch.""" +from sglang.srt.layers.layer_boundary.output import complete_owed from sglang.srt.layers.layer_boundary.residual import access from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.model_executor.forward_batch_info import PPProxyTensors @@ -135,13 +136,24 @@ def snapshot(hidden_states, forward_batch): def add_to_output(hidden_states, forward_batch, extra): - output, _ = access.add_to_output(hidden_states, stream_of(forward_batch), extra) - return output + """Complete the output before adding an extra contribution exactly once. + Leave its residual update for the next prepare call.""" + hidden_states = stream_of(forward_batch).complete(hidden_states) + hidden_states.add_(extra) + return hidden_states def fold(hidden_states, forward_batch): - output, _ = access.fold(hidden_states, stream_of(forward_batch)) - return output + """Finish a plain layer output and fold its residual into a complete value. + The following stage receives it with no outstanding residual addition.""" + stream = stream_of(forward_batch) + if stream.pending is not None and not stream.pending.update.is_plain_add: + raise NotImplementedError("fold requires a plain residual update") + hidden_states, residual = stream.export(hidden_states) + hidden_states = complete_owed(hidden_states) + if residual is not None: + hidden_states = hidden_states + residual + return stream.write(hidden_states) def set_written(hidden_states, forward_batch): diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 6a4d049b65..6068e8f209 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -449,6 +449,9 @@ def _is_static_per_tensor_fp8_linear(quant_method, linear) -> bool: class RMSNorm(BaseFusedOp): + # The projection that consumes this norm's output; see fuse_input_quant(). + _quant_linear: Optional[nn.Module] = None + def __init__( self, hidden_size: int, @@ -499,6 +502,12 @@ class RMSNorm(BaseFusedOp): if force_native: self._forward_method = self.forward_native + def fuse_input_quant(self, linear: nn.Module) -> None: + """Fuse the static per-tensor FP8 input quantization of ``linear``, the + projection that consumes this norm's output, whenever forward_cuda can. + The projection stays out of this module's children.""" + self.__dict__["_quant_linear"] = linear + def forward_cuda( self, x: torch.Tensor, @@ -506,6 +515,8 @@ class RMSNorm(BaseFusedOp): post_residual_addition: Optional[torch.Tensor] = None, quant_linear: Optional[nn.Module] = None, ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if quant_linear is None: + quant_linear = self._quant_linear if x.numel() == 0: if residual is not None: if post_residual_addition is not None: diff --git a/python/sglang/srt/models/jet_nemotron.py b/python/sglang/srt/models/jet_nemotron.py index e0a1e3f25f..d7b779aa70 100644 --- a/python/sglang/srt/models/jet_nemotron.py +++ b/python/sglang/srt/models/jet_nemotron.py @@ -466,8 +466,7 @@ class JetNemotronDecoderLayer(nn.Module): positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor | None]: + ) -> torch.Tensor: # Self Attention residual = hidden_states @@ -490,7 +489,29 @@ class JetNemotronDecoderLayer(nn.Module): hidden_states = residual + hidden_states - return hidden_states, None + return hidden_states + + +class JetNemotronModel(Qwen2Model): + """Each decoder layer adds its own residual, so the stack runs without a + residual stream and ends with the final norm alone.""" + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + for i in range(self.start_layer, self.end_layer): + hidden_states = self.layers[i](positions, hidden_states, forward_batch) + if hidden_states.shape[0] != 0: + hidden_states = self.norm(hidden_states) + return hidden_states class JetNemotronForCausalLM(nn.Module): @@ -505,7 +526,7 @@ class JetNemotronForCausalLM(nn.Module): self.config = config self.quant_config = quant_config - self.model = Qwen2Model( + self.model = JetNemotronModel( config, quant_config=quant_config, prefix=add_prefix("model", prefix), diff --git a/python/sglang/srt/models/mimo_mtp.py b/python/sglang/srt/models/mimo_mtp.py index 6910e123b1..8391b58699 100644 --- a/python/sglang/srt/models/mimo_mtp.py +++ b/python/sglang/srt/models/mimo_mtp.py @@ -8,6 +8,7 @@ import torch from torch import nn from transformers import PretrainedConfig +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -69,13 +70,14 @@ class MiMoMultiTokenPredictorLayer(nn.Module): ) ) - hidden_states, residual = self.mtp_block( + residual_batch.start(forward_batch) + hidden_states = self.mtp_block( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=None, ) - hidden_states = residual + hidden_states + hidden_states = residual_batch.fold(hidden_states, forward_batch) + hidden_states = residual_batch.take_output(hidden_states, forward_batch) hidden_states = self.final_layernorm(hidden_states) return hidden_states diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 6e0016d834..ff0241a528 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -26,12 +26,15 @@ from sglang.srt.distributed import ( get_pp_indices, ) from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList from sglang.srt.layers.dp_attention import is_dp_attention_enabled -from sglang.srt.layers.layer_boundary.residual.access import ( - export_output, - from_pp, - snapshot, +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, ) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -183,6 +186,7 @@ class Qwen2Attention(nn.Module): hidden_size, bias=False, quant_config=quant_config, + reduce_results=False, prefix=add_prefix("o_proj", prefix), ) @@ -280,29 +284,38 @@ class Qwen2DecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj) + self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj) + self.attn_boundary, self.ffn_boundary = make_stages( + (declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm), + ( + declare_ffn(sparse=False, next_layer_sparse=False), + self.post_attention_layernorm, + ), + previous=declare_ffn() if layer_id != 0 else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) + + def _attn_readout(self, layer_id: int): + return NORM_QUANT_READOUT def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: + capture_output=None, + ) -> torch.Tensor: # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm( - hidden_states, quant_linear=self.self_attn.qkv_proj - ) - else: - hidden_states, residual = self.input_layernorm( - hidden_states, residual, quant_linear=self.self_attn.qkv_proj - ) + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture=capture_output + ) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, @@ -310,11 +323,11 @@ class Qwen2DecoderLayer(nn.Module): ) # Fully Connected - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual, quant_linear=self.mlp.gate_up_proj - ) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class Qwen2Model(nn.Module): @@ -410,37 +423,29 @@ class Qwen2Model(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states, residual = from_pp(pp_proxy_tensors) + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): - if i in self.layers_to_capture: - aux_hidden_states.append(snapshot(hidden_states, residual)) - layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = self.layers[i]( positions, hidden_states, forward_batch, - residual, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) - hidden_states, residual = export_output(hidden_states, residual, forward_batch) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) - else: - if hidden_states.shape[0] != 0: - if residual is None: - hidden_states = self.norm(hidden_states) - else: - hidden_states, _ = self.norm(hidden_states, residual) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) if len(aux_hidden_states) == 0: return hidden_states @@ -578,6 +583,7 @@ class Qwen2ForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -585,19 +591,17 @@ class Qwen2ForCausalLM(nn.Module): # decoder layer for i in range(start, end): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( + forward_batch.hidden_states = layer( positions, forward_batch.hidden_states, forward_batch, - forward_batch.residual, ) if end == self.model.config.num_hidden_layers: # norm - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual + forward_batch.hidden_states = residual_batch.final_norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) - forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( input_ids, forward_batch.hidden_states, self.lm_head, forward_batch diff --git a/python/sglang/srt/models/qwen2_eagle.py b/python/sglang/srt/models/qwen2_eagle.py index f1842a2af4..51375ebea0 100644 --- a/python/sglang/srt/models/qwen2_eagle.py +++ b/python/sglang/srt/models/qwen2_eagle.py @@ -24,6 +24,8 @@ from typing import Iterable, Optional, Tuple import torch from torch import nn +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -51,7 +53,11 @@ class Qwen2DecoderLayer(Qwen2DecoderLayer): # https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427 if layer_id == 0: del self.input_layernorm - setattr(self, "input_layernorm", lambda x, quant_linear=None: x) + + def _attn_readout(self, layer_id: int): + if layer_id == 0: + return PLAIN_READOUT + return super()._attn_readout(layer_id) class Qwen2Model(nn.Module): @@ -99,16 +105,11 @@ class Qwen2Model(nn.Module): torch.cat((hidden_states, forward_batch.spec_info.hidden_states), dim=-1) ) - residual = None - for i in range(len(self.layers)): - layer = self.layers[i] - hidden_states, residual = layer( - positions, - hidden_states, - forward_batch, - residual, - ) - return hidden_states + residual + residual_batch.start(forward_batch) + for layer in self.layers: + hidden_states = layer(positions, hidden_states, forward_batch) + hidden_states = residual_batch.fold(hidden_states, forward_batch) + return residual_batch.take_output(hidden_states, forward_batch) class Qwen2ForCausalLMEagle(Qwen2ForCausalLM): diff --git a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py index c918c4cff8..8ba1fda633 100644 --- a/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py +++ b/test/registered/unit/layer_boundary/test_cutedsl_ar_fusion.py @@ -35,7 +35,7 @@ from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( prepare_cutedsl_fusion, ) from sglang.srt.layers.layer_boundary.prepare import _run_entry, _update_read -from sglang.srt.layers.layer_boundary.residual.access import export_output +from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.model_executor.cuda_graph_config import CudaGraphConfig, PhaseConfig from sglang.srt.model_executor.forward_batch_info import ForwardMode @@ -434,9 +434,9 @@ def test_the_layer_stack_hands_a_handoff_only_to_a_final_norm_that_takes_it(): for takes, completed in ((True, 0), (False, 1)): finish = MagicMock(return_value=torch.ones(2, 8)) handoff = _handoff(finish) - hidden, _ = export_output( - handoff, torch.zeros(2, 8), _DECODE, final_norm_takes_handoff=takes - ) + stream = ResidualStream(torch.zeros(2, 8)) + owed = stream.record(handoff, PLAIN_ADD) + hidden, _ = stream.export(owed, takes_handoff=takes) assert finish.call_count == completed assert (hidden is handoff) == takes diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 8f17a14ef3..5150d3b850 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -45,7 +45,6 @@ from sglang.srt.layers.layer_boundary.ops import ( update_attn_tp_gather_output, ) from sglang.srt.layers.layer_boundary.residual import mhc as mhc_module -from sglang.srt.layers.layer_boundary.residual.access import export_output from sglang.srt.layers.layer_boundary.residual.add_norm import ( NORM_READOUT, PLAIN_RESIDUAL_OPS, @@ -1860,7 +1859,7 @@ class TestTwoLayers(CustomTestCase): hidden = moe(hidden, s) hidden, residual = finish_exit(ffn_exit, hidden, residual) handed_on.append(type(hidden)) - hidden, residual = export_output(hidden, residual, forward_batch) + hidden, residual = residual.export(hidden) # The last a2a layer folds the residual into its output. if residual is not None: residual = residual[: s.rows] diff --git a/test/registered/unit/layer_boundary/test_ffn_exit.py b/test/registered/unit/layer_boundary/test_ffn_exit.py index 4a5aa4a3a2..e3e794e1f0 100644 --- a/test/registered/unit/layer_boundary/test_ffn_exit.py +++ b/test/registered/unit/layer_boundary/test_ffn_exit.py @@ -17,7 +17,6 @@ from sglang.srt.layers.layer_boundary import ops as transport_ops from sglang.srt.layers.layer_boundary import prepare as comm_ops from sglang.srt.layers.layer_boundary.contracts import BatchVariant from sglang.srt.layers.layer_boundary.ops import keep_output -from sglang.srt.layers.layer_boundary.residual.access import export_output from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.runtime_context import get_forward from sglang.test.boundary_fixtures import ( @@ -319,13 +318,10 @@ class TestReduceOutput(CustomTestCase): hidden_states = torch.ones(3, 4) hidden_states, _ = finish_exit(ffn_exit, hidden_states, residual) - hidden_states, residual_out = export_output( - hidden_states, residual, object() - ) + hidden_states = complete_owed(hidden_states) self.assertEqual(self.all_reduce.call_count, int(fuse)) expected = 3.0 if fuse else 2.0 # all-reduce stub / postprocess stub torch.testing.assert_close(hidden_states, torch.full((3, 4), expected)) - self.assertIs(residual_out, residual) class TestSelectFfnCompletion(CustomTestCase): diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index f2941cefc5..08dab8e8be 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -18,7 +18,7 @@ from sglang.srt.layers.layer_boundary import ( ) from sglang.srt.layers.layer_boundary.contracts import BatchVariant, StageKind from sglang.srt.layers.layer_boundary.output import UnreducedOutput -from sglang.srt.layers.layer_boundary.residual.access import add_to_output +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD from sglang.srt.layers.layer_boundary.residual.stream import ( DeclaredSum, @@ -354,12 +354,13 @@ class TestResidualStream(CustomTestCase): torch.testing.assert_close(outputs[0], torch.full((2, 4), 5.0)) def test_deepstack_adds_once_after_completing_the_sum(self): - hidden, stream = add_to_output( - self.hidden, self.stream, torch.full((2, 4), 7.0) + hidden = residual_batch.add_to_output( + self.hidden, + SimpleNamespace(residual_stream=self.stream), + torch.full((2, 4), 7.0), ) - self.assertIs(stream, self.stream) - self.assertIs(stream.pending.value, hidden) - self.assertIs(stream.residual, self.residual) + self.assertIs(self.stream.pending.value, hidden) + self.assertIs(self.stream.residual, self.residual) torch.testing.assert_close(hidden, torch.full((2, 4), 9.0)) self.group.all_reduce.assert_called_once()