[Memory] Share one full prefill graph output buffer across shapes for tuple and list outputs (#43176)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
842974287
parent
e14ef4b658
commit
cba1d07a94
@@ -48,15 +48,22 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.runner.shape_key import ShapeKey
|
||||
|
||||
|
||||
def _allocate_output_buffer(output: Any) -> Optional[torch.Tensor]:
|
||||
if not torch.is_tensor(output) or output.ndim == 0:
|
||||
return None
|
||||
return torch.empty_like(output)
|
||||
def _allocate_output_buffer(output: Any) -> Optional[Any]:
|
||||
"""Same-structure buffer for a tensor or a (nested) list/tuple of
|
||||
row-major tensors; None when any leaf is something else."""
|
||||
if torch.is_tensor(output):
|
||||
return None if output.ndim == 0 else torch.empty_like(output)
|
||||
if isinstance(output, (list, tuple)) and output:
|
||||
buffers = [_allocate_output_buffer(item) for item in output]
|
||||
if any(buffer is None for buffer in buffers):
|
||||
return None
|
||||
return tuple(buffers) if isinstance(output, tuple) else buffers
|
||||
return None
|
||||
|
||||
|
||||
def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool:
|
||||
def _output_fits_buffer(output: torch.Tensor, output_buffer: Any) -> bool:
|
||||
return (
|
||||
torch.is_tensor(output)
|
||||
torch.is_tensor(output_buffer)
|
||||
and output.ndim == output_buffer.ndim
|
||||
and output.shape[1:] == output_buffer.shape[1:]
|
||||
and output.shape[0] <= output_buffer.shape[0]
|
||||
@@ -65,14 +72,28 @@ def _output_fits_buffer(output: Any, output_buffer: torch.Tensor) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _copy_output_to_buffer(
|
||||
output: Any, output_buffer: torch.Tensor
|
||||
) -> Optional[torch.Tensor]:
|
||||
if not _output_fits_buffer(output, output_buffer):
|
||||
return None
|
||||
shared_output = output_buffer[: output.shape[0]]
|
||||
shared_output.copy_(output)
|
||||
return shared_output
|
||||
def _copy_output_to_buffer(output: Any, output_buffer: Any) -> Optional[Any]:
|
||||
"""Copy ``output`` leaf-wise into the leading rows of ``output_buffer`` and
|
||||
return the same structure of buffer views; None when it does not fit."""
|
||||
if torch.is_tensor(output):
|
||||
if not _output_fits_buffer(output, output_buffer):
|
||||
return None
|
||||
shared_output = output_buffer[: output.shape[0]]
|
||||
shared_output.copy_(output)
|
||||
return shared_output
|
||||
if isinstance(output, (list, tuple)):
|
||||
if not isinstance(output_buffer, (list, tuple)) or len(output) != len(
|
||||
output_buffer
|
||||
):
|
||||
return None
|
||||
shared = [
|
||||
_copy_output_to_buffer(item, buffer)
|
||||
for item, buffer in zip(output, output_buffer)
|
||||
]
|
||||
if any(item is None for item in shared):
|
||||
return None
|
||||
return tuple(shared) if isinstance(output, tuple) else shared
|
||||
return None
|
||||
|
||||
|
||||
class FullCudaGraphBackend(BaseCudaGraphBackend):
|
||||
@@ -96,7 +117,7 @@ class FullCudaGraphBackend(BaseCudaGraphBackend):
|
||||
self._capture_stream: Optional[torch.cuda.Stream] = None
|
||||
self._precarve = GraphPoolPrecarve()
|
||||
self._reuse_output_buffer = reuse_output_buffer
|
||||
self._output_buffer: Optional[torch.Tensor] = None
|
||||
self._output_buffer: Optional[Any] = None
|
||||
self._memory_saver_adapter: Optional[Any] = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
and get_bool_env_var("SGLANG_MEMORY_SAVER_CUDA_GRAPH")
|
||||
|
||||
Reference in New Issue
Block a user