[Memory] Share one full prefill graph output buffer across shapes for tuple and list outputs (#43176)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-08 23:05:29 -07:00
committed by GitHub
co-authored by Hanming Lu 842974287
parent e14ef4b658
commit cba1d07a94
2 changed files with 90 additions and 15 deletions
@@ -48,15 +48,22 @@ if TYPE_CHECKING:
from sglang.srt.model_executor.runner.shape_key import ShapeKey
def _allocate_output_buffer(output: Any) -> Optional[torch.Tensor]:
if not torch.is_tensor(output) or output.ndim == 0:
return None
return torch.empty_like(output)
def _allocate_output_buffer(output: Any) -> Optional[Any]:
"""Same-structure buffer for a tensor or a (nested) list/tuple of
row-major tensors; None when any leaf is something else."""
if torch.is_tensor(output):
return None if output.ndim == 0 else torch.empty_like(output)
if isinstance(output, (list, tuple)) and output:
buffers = [_allocate_output_buffer(item) for item in output]
if any(buffer is None for buffer in buffers):
return None
return tuple(buffers) if isinstance(output, tuple) else buffers
return None
def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool:
def _output_fits_buffer(output: torch.Tensor, output_buffer: Any) -> bool:
return (
torch.is_tensor(output)
torch.is_tensor(output_buffer)
and output.ndim == output_buffer.ndim
and output.shape[1:] == output_buffer.shape[1:]
and output.shape[0] <= output_buffer.shape[0]
@@ -65,14 +72,28 @@ def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool:
)
def _copy_output_to_buffer(
output: Any, output_buffer: torch.Tensor
) -> Optional[torch.Tensor]:
if not _output_fits_buffer(output, output_buffer):
return None
shared_output = output_buffer[: output.shape[0]]
shared_output.copy_(output)
return shared_output
def _copy_output_to_buffer(output: Any, output_buffer: Any) -> Optional[Any]:
"""Copy ``output`` leaf-wise into the leading rows of ``output_buffer`` and
return the same structure of buffer views; None when it does not fit."""
if torch.is_tensor(output):
if not _output_fits_buffer(output, output_buffer):
return None
shared_output = output_buffer[: output.shape[0]]
shared_output.copy_(output)
return shared_output
if isinstance(output, (list, tuple)):
if not isinstance(output_buffer, (list, tuple)) or len(output) != len(
output_buffer
):
return None
shared = [
_copy_output_to_buffer(item, buffer)
for item, buffer in zip(output, output_buffer)
]
if any(item is None for item in shared):
return None
return tuple(shared) if isinstance(output, tuple) else shared
return None
class FullCudaGraphBackend(BaseCudaGraphBackend):
@@ -96,7 +117,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
self._capture_stream: Optional[torch.cuda.Stream] = None
self._precarve = GraphPoolPrecarve()
self._reuse_output_buffer = reuse_output_buffer
self._output_buffer: Optional[torch.Tensor] = None
self._output_buffer: Optional[Any] = None
self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
and get_bool_env_var("SGLANG_MEMORY_SAVER_CUDA_GRAPH")