From 8d2d87475832c31f4d92df9e5c5616f36393cbd3 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:38:05 -0700 Subject: [PATCH] [Fix] Release consumed residual contributions (#41749) --- .../layers/layer_boundary/residual/stream.py | 12 ++++++- .../layer_boundary/test_residual_stream.py | 34 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/layer_boundary/residual/stream.py b/python/sglang/srt/layers/layer_boundary/residual/stream.py index eeb13d7bed..34638f56a8 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/stream.py +++ b/python/sglang/srt/layers/layer_boundary/residual/stream.py @@ -86,6 +86,11 @@ class Contribution(msgspec.Struct): self.value, self.owed = value, None return value + def release(self): + """Drop the tensors of a contribution its consumer has taken.""" + self.value = None + self.owed = None + class OwedOutput(msgspec.Struct, frozen=True): """Opaque model-facing handle. Only its boundary may read the contribution.""" @@ -105,7 +110,8 @@ class ResidualStream: field separately records incomplete reduction/finalize work. The states are initial (both None), written (residual only), and pending. - Prepare consumes pending once and writes the resulting residual. Completing + Prepare consumes pending once and writes the resulting residual, which + releases the consumed contribution's tensors. Completing communication for capture leaves the update pending. Layouts and batch-path selection live on the bound stage, not on tensor values. """ @@ -136,6 +142,10 @@ class ResidualStream: return stream.leave(hidden, update, declared_sum=declared_sum), stream def write(self, residual): + if self.pending is not None: + # The written residual consumes the pending contribution. Handles + # to it are now stale and must not keep its tensors alive. + self.pending.release() self.residual = residual self.pending = None return residual diff --git a/test/registered/unit/layer_boundary/test_residual_stream.py b/test/registered/unit/layer_boundary/test_residual_stream.py index dc8e8b31ba..0cf4e556bf 100644 --- a/test/registered/unit/layer_boundary/test_residual_stream.py +++ b/test/registered/unit/layer_boundary/test_residual_stream.py @@ -385,6 +385,40 @@ class TestResidualStream(CustomTestCase): self.assertIs(stream.pending.value, completed) move.assert_called_once_with(self.partial) + def test_prepare_releases_the_consumed_contribution(self): + class Read: + norms_plainly = True + + def update_and_read(self, update, value, residual, norm, **kwargs): + residual = update.update(value, residual) + return residual * 2, residual + + rows = Layout(frozenset()) + boundary = make_boundary( + EdgeDecl(StageOutput(rows), StageInput(rows, read=Read()), rows, rows) + ) + steps = StageSteps(StageEntry(boundary.prepare, rows), StageOutput(rows), None) + stage = StageBoundary( + SimpleNamespace(norm=None), + declaration=SimpleNamespace(kind=StageKind.ATTENTION), + ) + partial = torch.ones(4, 4) + partial_ref = weakref.ref(partial) + stream = ResidualStream(torch.full((2, 4), 3.0)) + # The deferred DP completion: sum, then this rank's rows. + hidden = stream.leave( + UnreducedOutput(partial, reduce_and_redistribute=lambda x: x[:2] * 2), + ADD, + ) + del partial + result, stream = stage._prepare(hidden, stream, None, steps) + torch.testing.assert_close(result, torch.full((2, 4), 10.0)) + # The caller still holds the handle; it no longer keeps the partial. + self.assertIsInstance(hidden, OwedOutput) + self.assertIsNone(partial_ref()) + with self.assertRaises(RuntimeError): + stream.input(hidden) + class TestBatchStageOwnership(CustomTestCase): def test_terminal_norm_releases_layer_buffers(self):