From cba1d07a94750bc0f06034917293c1cbf9510df8 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 8 Oct 2026 23:05:29 -0700 Subject: [PATCH] [Memory] Share one full prefill graph output buffer across shapes for tuple and list outputs (#43176) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com> --- .../runner_backend/full_cuda_graph_backend.py | 51 ++++++++++++------ .../test_full_cuda_graph_backend.py | 54 +++++++++++++++++++ 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py index a72528c612..dddf37fcd4 100644 --- a/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/full_cuda_graph_backend.py @@ -48,15 +48,22 @@ if TYPE_CHECKING: from sglang.srt.model_executor.runner.shape_key import ShapeKey -def _allocate_output_buffer(output: Any) -> Optional[torch.Tensor]: - if not torch.is_tensor(output) or output.ndim == 0: - return None - return torch.empty_like(output) +def _allocate_output_buffer(output: Any) -> Optional[Any]: + """Same-structure buffer for a tensor or a (nested) list/tuple of + row-major tensors; None when any leaf is something else.""" + if torch.is_tensor(output): + return None if output.ndim == 0 else torch.empty_like(output) + if isinstance(output, (list, tuple)) and output: + buffers = [_allocate_output_buffer(item) for item in output] + if any(buffer is None for buffer in buffers): + return None + return tuple(buffers) if isinstance(output, tuple) else buffers + return None -def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool: +def _output_fits_buffer(output: torch.Tensor, output_buffer: Any) -> bool: return ( - torch.is_tensor(output) + torch.is_tensor(output_buffer) and output.ndim == output_buffer.ndim and output.shape[1:] == output_buffer.shape[1:] and output.shape[0] <= output_buffer.shape[0] @@ -65,14 +72,28 @@ def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool: ) -def _copy_output_to_buffer( - output: Any, output_buffer: torch.Tensor -) -> Optional[torch.Tensor]: - if not _output_fits_buffer(output, output_buffer): - return None - shared_output = output_buffer[: output.shape[0]] - shared_output.copy_(output) - return shared_output +def _copy_output_to_buffer(output: Any, output_buffer: Any) -> Optional[Any]: + """Copy ``output`` leaf-wise into the leading rows of ``output_buffer`` and + return the same structure of buffer views; None when it does not fit.""" + if torch.is_tensor(output): + if not _output_fits_buffer(output, output_buffer): + return None + shared_output = output_buffer[: output.shape[0]] + shared_output.copy_(output) + return shared_output + if isinstance(output, (list, tuple)): + if not isinstance(output_buffer, (list, tuple)) or len(output) != len( + output_buffer + ): + return None + shared = [ + _copy_output_to_buffer(item, buffer) + for item, buffer in zip(output, output_buffer) + ] + if any(item is None for item in shared): + return None + return tuple(shared) if isinstance(output, tuple) else shared + return None class FullCudaGraphBackend(BaseCudaGraphBackend): @@ -96,7 +117,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend): self._capture_stream: Optional[torch.cuda.Stream] = None self._precarve = GraphPoolPrecarve() self._reuse_output_buffer = reuse_output_buffer - self._output_buffer: Optional[torch.Tensor] = None + self._output_buffer: Optional[Any] = None self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create( enable=enable_memory_saver and get_bool_env_var("SGLANG_MEMORY_SAVER_CUDA_GRAPH") diff --git a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py index 6820ef21bd..8a3d88887f 100644 --- a/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py +++ b/test/registered/unit/model_executor/runner_backend/test_full_cuda_graph_backend.py @@ -135,6 +135,60 @@ class TestCaptureOneNoProfiling(CustomTestCase): self.assertEqual(large.data_ptr(), small.data_ptr()) self.assertEqual(backend._output_buffer.shape, (4, 2)) + def test_prefill_tuple_outputs_share_one_output_storage(self): + # A model whose forward returns a tuple of row-major tensors shares one + # buffer per leaf, and a later output that does not match the buffer's + # structure keeps its own storage without disturbing the shared views. + runner = _make_runner(enable_profile=False, profiler=None, mode_name="EXTEND") + backend = _make_backend(runner) + backend._reuse_output_buffer = True + + large_out = (torch.ones((4, 2)), torch.ones((4, 3))) + small_out = (torch.full((2, 2), 2.0), torch.full((2, 3), 3.0)) + plain_out = torch.full((1, 2), 7.0) + outputs = iter([large_out] * 3 + [small_out] * 3 + [plain_out] * 3) + with mock.patch("torch.cuda.CUDAGraph", side_effect=["G4", "G2", "G1"]): + backend.capture_one(ShapeKey(size=4), lambda: next(outputs)) + backend.capture_one(ShapeKey(size=2), lambda: next(outputs)) + backend.capture_one(ShapeKey(size=1), lambda: next(outputs)) + + large = backend._outputs[ShapeKey(size=4)] + small = backend._outputs[ShapeKey(size=2)] + self.assertIsInstance(large, tuple) + self.assertIsInstance(small, tuple) + self.assertEqual([t.shape for t in large], [(4, 2), (4, 3)]) + self.assertEqual([t.shape for t in small], [(2, 2), (2, 3)]) + for large_leaf, small_leaf, buffer_leaf in zip( + large, small, backend._output_buffer + ): + self.assertEqual(large_leaf.data_ptr(), small_leaf.data_ptr()) + self.assertEqual(large_leaf.data_ptr(), buffer_leaf.data_ptr()) + self.assertTrue(torch.equal(small[0], small_out[0])) + self.assertTrue(torch.equal(small[1], small_out[1])) + # The plain tensor does not fit a tuple buffer: it is kept as-is and + # sharing is switched off for the rest of the capture. + self.assertIs(backend._outputs[ShapeKey(size=1)], plain_out) + self.assertFalse(backend._reuse_output_buffer) + + def test_prefill_tuple_with_scalar_leaf_keeps_independent_storage(self): + # A zero-dim leaf has no leading row axis to share, so the whole tuple + # keeps independent storage instead of allocating a partial buffer. + runner = _make_runner(enable_profile=False, profiler=None, mode_name="EXTEND") + backend = _make_backend(runner) + backend._reuse_output_buffer = True + + first_out = (torch.ones((4, 2)), torch.tensor(0.0)) + second_out = (torch.ones((2, 2)), torch.tensor(1.0)) + outputs = iter([first_out] * 3 + [second_out] * 3) + with mock.patch("torch.cuda.CUDAGraph", side_effect=["G4", "G2"]): + backend.capture_one(ShapeKey(size=4), lambda: next(outputs)) + backend.capture_one(ShapeKey(size=2), lambda: next(outputs)) + + self.assertIsNone(backend._output_buffer) + self.assertFalse(backend._reuse_output_buffer) + self.assertIs(backend._outputs[ShapeKey(size=4)], first_out) + self.assertIs(backend._outputs[ShapeKey(size=2)], second_out) + def test_enable_flag_set_but_no_profiler_attr_does_not_step(self): # The runner advertises the flag but never created a profiler; the # getattr guard must keep capture_one on the non-profiling path.